1532 lines
52 KiB
Python
1532 lines
52 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
"""Generate optional vLLM benchmark sweep files from one initial suggestion."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import shlex
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
from .runtime_tuning import WorkloadHints
|
|
|
|
SUPPORTED_TENSOR_PARALLEL_SIZES = frozenset({1, 2, 4, 8})
|
|
|
|
MOE_EXPERT_COUNT_KEYS = (
|
|
"num_experts",
|
|
"num_local_experts",
|
|
"num_sparse_experts",
|
|
"num_total_experts",
|
|
"n_experts",
|
|
"n_routed_experts",
|
|
"num_routed_experts",
|
|
)
|
|
MOE_INTERMEDIATE_SIZE_KEYS = (
|
|
"moe_intermediate_size",
|
|
"expert_intermediate_size",
|
|
"moe_ffn_hidden_size",
|
|
)
|
|
|
|
PROMPTS_PER_CONCURRENCY = 10
|
|
MIN_NUM_PROMPTS = 100
|
|
MAX_NUM_PROMPTS = 1000
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ModelParallelMetadata:
|
|
"""Model dimensions needed to reject invalid MoE layouts before startup."""
|
|
|
|
is_moe: bool | None
|
|
num_experts: int | None = None
|
|
moe_intermediate_size: int | None = None
|
|
|
|
|
|
UNKNOWN_MODEL_PARALLEL_METADATA = ModelParallelMetadata(is_moe=None)
|
|
|
|
|
|
def _num_prompts_for_concurrency(concurrency: int) -> int:
|
|
return min(
|
|
MAX_NUM_PROMPTS,
|
|
max(MIN_NUM_PROMPTS, concurrency * PROMPTS_PER_CONCURRENCY),
|
|
)
|
|
|
|
|
|
def _positive_int(config: dict[str, Any], key: str) -> int:
|
|
value = config.get(key)
|
|
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
|
raise ValueError(f"Sweep generation requires a positive initial {key!r} value.")
|
|
return value
|
|
|
|
|
|
def _strict_lower_power_of_two(value: int) -> int:
|
|
if value <= 1:
|
|
return 1
|
|
return 1 << ((value - 1).bit_length() - 1)
|
|
|
|
|
|
def _strict_upper_power_of_two(value: int) -> int:
|
|
return 1 << value.bit_length()
|
|
|
|
|
|
def validate_sweep_workload(workload: WorkloadHints) -> None:
|
|
required = {
|
|
"--input-tokens": workload.input_tokens,
|
|
"--output-tokens": workload.output_tokens,
|
|
"--concurrency": workload.concurrency,
|
|
}
|
|
missing = [name for name, value in required.items() if value is None]
|
|
if missing:
|
|
raise ValueError("--generate-sweep requires " + ", ".join(missing) + ".")
|
|
|
|
|
|
def build_serve_params(config: dict[str, Any]) -> list[dict[str, Any]]:
|
|
"""Build a bounded batch-budget sweep at the sequence-count lower bound.
|
|
|
|
Explicit candidates keep the initial per-replica sequence count. Three
|
|
reference points compare those tuned values with vLLM-resolved defaults.
|
|
"""
|
|
initial_seqs = _positive_int(config, "max-num-seqs")
|
|
initial_batch = _positive_int(config, "max-num-batched-tokens")
|
|
|
|
minimum_batch = initial_seqs
|
|
if config.get("enable-chunked-prefill") is False:
|
|
max_model_len = config.get("max-model-len")
|
|
if (
|
|
isinstance(max_model_len, int)
|
|
and not isinstance(max_model_len, bool)
|
|
and max_model_len > 0
|
|
):
|
|
minimum_batch = max(minimum_batch, max_model_len)
|
|
|
|
lower_batch = max(
|
|
minimum_batch,
|
|
_strict_lower_power_of_two(initial_batch),
|
|
)
|
|
smaller_batch = max(
|
|
minimum_batch,
|
|
_strict_lower_power_of_two(lower_batch),
|
|
)
|
|
higher_batch = max(
|
|
minimum_batch,
|
|
_strict_upper_power_of_two(initial_batch),
|
|
)
|
|
|
|
candidates: list[dict[str, Any]] = []
|
|
seen: set[tuple[int | None, int | None]] = set()
|
|
|
|
def add(
|
|
name: str,
|
|
max_num_seqs: int | None,
|
|
max_num_batched_tokens: int | None,
|
|
) -> None:
|
|
if max_num_seqs is not None and max_num_batched_tokens is not None:
|
|
max_num_batched_tokens = max(max_num_batched_tokens, max_num_seqs)
|
|
signature = (max_num_seqs, max_num_batched_tokens)
|
|
if signature in seen:
|
|
return
|
|
seen.add(signature)
|
|
|
|
candidate: dict[str, Any] = {"_benchmark_name": name}
|
|
if max_num_seqs is not None:
|
|
candidate["max_num_seqs"] = max_num_seqs
|
|
if max_num_batched_tokens is not None:
|
|
candidate["max_num_batched_tokens"] = max_num_batched_tokens
|
|
candidates.append(candidate)
|
|
|
|
# Keep the exact initial suggestion as the measured baseline. Additional
|
|
# values exist only in the optional sweep package.
|
|
add("initial", initial_seqs, initial_batch)
|
|
add("smaller_batch_budget", initial_seqs, smaller_batch)
|
|
add("lower_batch_budget", initial_seqs, lower_batch)
|
|
add("higher_batch_budget", initial_seqs, higher_batch)
|
|
# Default-reference candidates intentionally omit one or both scheduler
|
|
# keys. The sweep server starts from sweep_config.yml, where both keys are
|
|
# removed, so omission lets vLLM resolve its normal runtime default.
|
|
add("vllm_default_max_num_seqs", None, initial_batch)
|
|
add("vllm_default_max_num_batched_tokens", initial_seqs, None)
|
|
add("vllm_defaults", None, None)
|
|
|
|
return candidates
|
|
|
|
|
|
def _scheduler_baseline_for_dp(
|
|
config: dict[str, Any], workload: WorkloadHints, data_parallel_size: int
|
|
) -> tuple[int, int]:
|
|
"""Return per-replica scheduler settings for a DP layout."""
|
|
assert workload.concurrency is not None
|
|
assert workload.input_tokens is not None
|
|
per_replica_concurrency = math.ceil(workload.concurrency / data_parallel_size)
|
|
output_tokens = workload.output_tokens or 1
|
|
prefills_per_step = max(1.0, per_replica_concurrency / output_tokens)
|
|
if workload.target_qps is not None and workload.tpot_sla_ms is not None:
|
|
per_replica_qps = workload.target_qps / data_parallel_size
|
|
prefills_per_step = max(
|
|
prefills_per_step,
|
|
per_replica_qps * workload.tpot_sla_ms / 1000.0,
|
|
)
|
|
prefills_per_step = min(float(per_replica_concurrency), prefills_per_step)
|
|
batch = max(
|
|
2048,
|
|
per_replica_concurrency,
|
|
per_replica_concurrency + math.ceil(workload.input_tokens * prefills_per_step),
|
|
)
|
|
if config.get("enable-chunked-prefill") is False:
|
|
max_model_len = config.get("max-model-len")
|
|
if isinstance(max_model_len, int) and not isinstance(max_model_len, bool):
|
|
batch = max(batch, max_model_len)
|
|
return per_replica_concurrency, batch
|
|
|
|
|
|
def _model_config_value(view: Any, key: str) -> Any:
|
|
if isinstance(view, dict):
|
|
return view.get(key)
|
|
return getattr(view, key, None)
|
|
|
|
|
|
def _config_views(hf_config: Any) -> list[Any]:
|
|
"""Return common nested model config views without duplicates."""
|
|
views = [hf_config]
|
|
for view in views:
|
|
for key in (
|
|
"text_config",
|
|
"language_config",
|
|
"llm_config",
|
|
"ffn_config",
|
|
"moe_config",
|
|
):
|
|
value = _model_config_value(view, key)
|
|
if value is not None and all(value is not item for item in views):
|
|
views.append(value)
|
|
return views
|
|
|
|
|
|
def _first_positive_model_int(views: list[Any], keys: tuple[str, ...]) -> int | None:
|
|
for view in views:
|
|
for key in keys:
|
|
value = _model_config_value(view, key)
|
|
if isinstance(value, int) and not isinstance(value, bool) and value > 0:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _config_identifies_moe(views: list[Any]) -> bool:
|
|
for view in views:
|
|
model_type = _model_config_value(view, "model_type")
|
|
if isinstance(model_type, str) and "moe" in model_type.lower():
|
|
return True
|
|
architectures = _model_config_value(view, "architectures")
|
|
if isinstance(architectures, list) and any(
|
|
isinstance(name, str) and "moe" in name.lower() for name in architectures
|
|
):
|
|
return True
|
|
return False
|
|
|
|
|
|
def model_parallel_metadata_from_config(hf_config: Any) -> ModelParallelMetadata:
|
|
"""Extract MoE markers from a loaded Transformers-compatible config."""
|
|
views = _config_views(hf_config)
|
|
num_experts = _first_positive_model_int(views, MOE_EXPERT_COUNT_KEYS)
|
|
moe_intermediate_size = _first_positive_model_int(views, MOE_INTERMEDIATE_SIZE_KEYS)
|
|
is_moe = (
|
|
num_experts is not None
|
|
or moe_intermediate_size is not None
|
|
or _config_identifies_moe(views)
|
|
)
|
|
if is_moe and moe_intermediate_size is None:
|
|
# Some MoE configs use the generic name for each expert's FFN width.
|
|
moe_intermediate_size = _first_positive_model_int(views, ("intermediate_size",))
|
|
return ModelParallelMetadata(
|
|
is_moe=is_moe,
|
|
num_experts=num_experts,
|
|
moe_intermediate_size=moe_intermediate_size,
|
|
)
|
|
|
|
|
|
def inspect_model_parallel_metadata(config: dict[str, Any]) -> ModelParallelMetadata:
|
|
"""Load the model config and extract portable MoE topology markers."""
|
|
model = config.get("model")
|
|
if not isinstance(model, str) or not model:
|
|
raise ValueError("Parallel-layout sweep requires a model in config.yml.")
|
|
|
|
from vllm.transformers_utils.config import get_config
|
|
|
|
hf_config = get_config(
|
|
model,
|
|
trust_remote_code=config.get("trust-remote-code") is True,
|
|
revision=config.get("revision"),
|
|
code_revision=config.get("code-revision"),
|
|
config_format=config.get("config-format", "auto"),
|
|
)
|
|
return model_parallel_metadata_from_config(hf_config)
|
|
|
|
|
|
def cpu_supports_moe_dp_collectives() -> bool:
|
|
"""Whether CPU overrides the variable-size collectives used by MoE DP."""
|
|
try:
|
|
from vllm.distributed.device_communicators.base_device_communicator import (
|
|
DeviceCommunicatorBase,
|
|
)
|
|
from vllm.distributed.device_communicators.cpu_communicator import (
|
|
CpuCommunicator,
|
|
)
|
|
except Exception:
|
|
return False
|
|
|
|
return all(
|
|
getattr(CpuCommunicator, name) is not getattr(DeviceCommunicatorBase, name)
|
|
for name in ("all_gatherv", "reduce_scatterv")
|
|
)
|
|
|
|
|
|
def _parallel_layout_name(
|
|
tensor_parallel_size: int, data_parallel_size: int, numa_node_count: int
|
|
) -> str:
|
|
name = f"tp{tensor_parallel_size}_dp{data_parallel_size}"
|
|
used_nodes = tensor_parallel_size * data_parallel_size
|
|
if used_nodes != numa_node_count:
|
|
name += f"_numa{used_nodes}of{numa_node_count}"
|
|
return name
|
|
|
|
|
|
def _default_parallel_layout_sizes(numa_node_count: int) -> list[tuple[int, int]]:
|
|
max_supported_tp = max(
|
|
size for size in SUPPORTED_TENSOR_PARALLEL_SIZES if size <= numa_node_count
|
|
)
|
|
layouts = []
|
|
for tensor_parallel_size in range(numa_node_count, 0, -1):
|
|
if tensor_parallel_size not in SUPPORTED_TENSOR_PARALLEL_SIZES:
|
|
continue
|
|
uses_all_numa_nodes = numa_node_count % tensor_parallel_size == 0
|
|
if not uses_all_numa_nodes and tensor_parallel_size != max_supported_tp:
|
|
continue
|
|
layouts.append(
|
|
(tensor_parallel_size, max(1, numa_node_count // tensor_parallel_size))
|
|
)
|
|
return layouts
|
|
|
|
|
|
def build_parallel_layout_plan(
|
|
config: dict[str, Any],
|
|
workload: WorkloadHints,
|
|
numa_node_count: int,
|
|
*,
|
|
model_metadata: ModelParallelMetadata = UNKNOWN_MODEL_PARALLEL_METADATA,
|
|
cpu_moe_dp_supported: bool = True,
|
|
tune_scheduler: bool = False,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
"""Build candidates and diagnostics after applying parallel safety policy."""
|
|
validate_sweep_workload(workload)
|
|
if numa_node_count <= 1:
|
|
raise ValueError("Parallel-layout sweep requires at least two NUMA nodes.")
|
|
|
|
layouts = _default_parallel_layout_sizes(numa_node_count)
|
|
skipped: list[dict[str, Any]] = []
|
|
if model_metadata.is_moe and not cpu_moe_dp_supported:
|
|
for tensor_parallel_size, data_parallel_size in layouts:
|
|
if data_parallel_size <= 1:
|
|
continue
|
|
skipped.append(
|
|
{
|
|
"_benchmark_name": _parallel_layout_name(
|
|
tensor_parallel_size, data_parallel_size, numa_node_count
|
|
),
|
|
"policy": "cpu_moe_dp_collectives",
|
|
"tensor_parallel_size": tensor_parallel_size,
|
|
"data_parallel_size": data_parallel_size,
|
|
"reason": (
|
|
"CPU communicator does not implement the variable-size "
|
|
"collectives required by MoE data parallelism"
|
|
),
|
|
}
|
|
)
|
|
# Keep TP coverage instead of collapsing the sweep to whichever DP=1
|
|
# layout happened to exist in the full-NUMA candidate set.
|
|
layouts = [
|
|
(tensor_parallel_size, 1)
|
|
for tensor_parallel_size in sorted(
|
|
SUPPORTED_TENSOR_PARALLEL_SIZES, reverse=True
|
|
)
|
|
if tensor_parallel_size <= numa_node_count
|
|
]
|
|
|
|
pcp_size = config.get("prefill-context-parallel-size", 1)
|
|
if not isinstance(pcp_size, int) or isinstance(pcp_size, bool) or pcp_size <= 0:
|
|
raise ValueError("prefill-context-parallel-size must be a positive integer.")
|
|
uses_expert_parallel = config.get("enable-expert-parallel") is True
|
|
|
|
candidates = []
|
|
for tensor_parallel_size, data_parallel_size in layouts:
|
|
name = _parallel_layout_name(
|
|
tensor_parallel_size, data_parallel_size, numa_node_count
|
|
)
|
|
effective_parallel_size = tensor_parallel_size * data_parallel_size * pcp_size
|
|
moe_width = model_metadata.moe_intermediate_size
|
|
if (
|
|
model_metadata.is_moe
|
|
and not uses_expert_parallel
|
|
and moe_width is not None
|
|
and moe_width % effective_parallel_size != 0
|
|
):
|
|
skipped.append(
|
|
{
|
|
"_benchmark_name": name,
|
|
"policy": "moe_effective_parallel_size",
|
|
"tensor_parallel_size": tensor_parallel_size,
|
|
"data_parallel_size": data_parallel_size,
|
|
"prefill_context_parallel_size": pcp_size,
|
|
"effective_parallel_size": effective_parallel_size,
|
|
"moe_intermediate_size": moe_width,
|
|
"reason": (
|
|
"moe_intermediate_size is not divisible by the effective "
|
|
"TP x DP x PCP size while expert parallelism is disabled"
|
|
),
|
|
}
|
|
)
|
|
continue
|
|
|
|
candidate = {
|
|
"_benchmark_name": name,
|
|
"tensor_parallel_size": tensor_parallel_size,
|
|
"data_parallel_size": data_parallel_size,
|
|
}
|
|
if tune_scheduler:
|
|
max_num_seqs, max_num_batched_tokens = _scheduler_baseline_for_dp(
|
|
config, workload, data_parallel_size
|
|
)
|
|
candidate["max_num_seqs"] = max_num_seqs
|
|
candidate["max_num_batched_tokens"] = max_num_batched_tokens
|
|
candidates.append(candidate)
|
|
|
|
if not candidates:
|
|
raise ValueError(
|
|
"No valid parallel-layout candidates remain after applying model and "
|
|
"runtime capability checks. See parallel_layout_skips.json."
|
|
)
|
|
return candidates, skipped
|
|
|
|
|
|
def build_parallel_layout_params(
|
|
config: dict[str, Any],
|
|
workload: WorkloadHints,
|
|
numa_node_count: int,
|
|
*,
|
|
model_metadata: ModelParallelMetadata = UNKNOWN_MODEL_PARALLEL_METADATA,
|
|
cpu_moe_dp_supported: bool = True,
|
|
tune_scheduler: bool = False,
|
|
) -> list[dict[str, Any]]:
|
|
"""Build full-NUMA layouts plus the largest supported TP layout."""
|
|
candidates, _ = build_parallel_layout_plan(
|
|
config,
|
|
workload,
|
|
numa_node_count,
|
|
model_metadata=model_metadata,
|
|
cpu_moe_dp_supported=cpu_moe_dp_supported,
|
|
tune_scheduler=tune_scheduler,
|
|
)
|
|
return candidates
|
|
|
|
|
|
def build_bench_params(workload: WorkloadHints) -> list[dict[str, Any]]:
|
|
validate_sweep_workload(workload)
|
|
assert workload.input_tokens is not None
|
|
assert workload.output_tokens is not None
|
|
assert workload.concurrency is not None
|
|
|
|
return [
|
|
{
|
|
"random_input_len": workload.input_tokens,
|
|
"random_output_len": workload.output_tokens,
|
|
"max_concurrency": workload.concurrency,
|
|
"num_prompts": _num_prompts_for_concurrency(workload.concurrency),
|
|
}
|
|
]
|
|
|
|
|
|
def _benchmark_models(config: dict[str, Any]) -> tuple[str, str]:
|
|
model = config.get("model")
|
|
if not isinstance(model, str) or not model:
|
|
raise ValueError("Sweep generation requires a model in config.yml.")
|
|
|
|
served_model_name = config.get("served-model-name")
|
|
if isinstance(served_model_name, str) and served_model_name:
|
|
return served_model_name, model
|
|
if (
|
|
isinstance(served_model_name, list)
|
|
and served_model_name
|
|
and isinstance(served_model_name[0], str)
|
|
):
|
|
return served_model_name[0], model
|
|
|
|
return model, model
|
|
|
|
|
|
def _relative_to(directory: Path, target: str) -> str:
|
|
return os.path.relpath(Path(target).resolve(), directory.resolve())
|
|
|
|
|
|
def _write_json(path: Path, value: object) -> None:
|
|
path.write_text(
|
|
json.dumps(value, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def _write_sweep_config(path: Path, config: dict[str, Any]) -> None:
|
|
sweep_config = dict(config)
|
|
sweep_config.pop("max-num-seqs", None)
|
|
sweep_config.pop("max-num-batched-tokens", None)
|
|
|
|
body = yaml.safe_dump(
|
|
sweep_config,
|
|
sort_keys=False,
|
|
default_flow_style=False,
|
|
allow_unicode=True,
|
|
)
|
|
header = (
|
|
"# Generated for runtime sweep only.\n"
|
|
"# Scheduler values are supplied per candidate; omitted values use "
|
|
"vLLM defaults.\n"
|
|
)
|
|
path.write_text(header + body, encoding="utf-8")
|
|
|
|
|
|
def _write_run_script(
|
|
path: Path,
|
|
*,
|
|
config_rel: str,
|
|
env_rel: str,
|
|
request_model: str,
|
|
tokenizer: str,
|
|
workload: WorkloadHints,
|
|
serve_params_name: str = "serve_params.json",
|
|
experiment_name: str = "runtime-tuning",
|
|
prepare_config_rel: str | None = None,
|
|
) -> None:
|
|
bench_parts = [
|
|
"vllm bench serve",
|
|
"--backend vllm",
|
|
f"--model {shlex.quote(request_model)}",
|
|
f"--tokenizer {shlex.quote(tokenizer)}",
|
|
"--dataset-name random",
|
|
"--request-rate inf",
|
|
"--ignore-eos",
|
|
"--metric-percentiles 99",
|
|
]
|
|
|
|
goodput_pairs: list[str] = []
|
|
if workload.ttft_sla_ms is not None:
|
|
goodput_pairs.append(f"ttft:{workload.ttft_sla_ms:g}")
|
|
if workload.tpot_sla_ms is not None:
|
|
goodput_pairs.append(f"tpot:{workload.tpot_sla_ms:g}")
|
|
if goodput_pairs:
|
|
bench_parts.append(
|
|
"--goodput " + " ".join(shlex.quote(pair) for pair in goodput_pairs)
|
|
)
|
|
|
|
bench_cmd = " ".join(bench_parts)
|
|
prepare_config = ""
|
|
if prepare_config_rel is not None:
|
|
prepare_config = f"""
|
|
SOURCE_CONFIG_PATH="${{SCRIPT_DIR}}/{prepare_config_rel}"
|
|
python3 - "${{SOURCE_CONFIG_PATH}}" "${{CONFIG_PATH}}" <<'PY'
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
source = Path(sys.argv[1])
|
|
target = Path(sys.argv[2])
|
|
config = yaml.safe_load(source.read_text(encoding="utf-8"))
|
|
if not isinstance(config, dict):
|
|
raise SystemExit(str(source) + " does not contain a YAML configuration object.")
|
|
config.pop("max-num-seqs", None)
|
|
config.pop("max-num-batched-tokens", None)
|
|
target.write_text(
|
|
yaml.safe_dump(
|
|
config,
|
|
sort_keys=False,
|
|
default_flow_style=False,
|
|
allow_unicode=True,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
PY
|
|
|
|
"""
|
|
|
|
script = f"""#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${{BASH_SOURCE[0]}}")" && pwd)"
|
|
CONFIG_PATH="${{SCRIPT_DIR}}/{config_rel}"
|
|
ENV_PATH="${{SCRIPT_DIR}}/{env_rel}"
|
|
|
|
source "${{ENV_PATH}}"
|
|
|
|
# Released/nightly vLLM images keep recipe-relative assets such as
|
|
# examples/*.jinja under /vllm-workspace. Launch sweep server subprocesses
|
|
# from there when available so relative recipe assets resolve like they do
|
|
# in serve_with_recipe.sh.
|
|
if [[ -d /vllm-workspace/examples ]]; then
|
|
cd /vllm-workspace
|
|
fi
|
|
|
|
# recipe tools may be newer than the installed vLLM package. Only pass the
|
|
# resilience flag when this vLLM CLI actually supports it.
|
|
CONTINUE_ON_ERROR_ARG=""
|
|
if vllm bench sweep serve --help 2>&1 | grep -q -- "--continue-on-error"; then
|
|
CONTINUE_ON_ERROR_ARG="--continue-on-error"
|
|
fi
|
|
|
|
# On older vLLM versions, a failed benchmark aborts the sweep process. Retry the
|
|
# interrupted sweep with --resume so one transient run failure does not discard
|
|
# completed work or force a manual restart.
|
|
SWEEP_RETRY_COUNT="${{VLLM_RECIPE_SWEEP_RETRY_COUNT:-0}}"
|
|
MAX_SWEEP_RETRIES="${{VLLM_RECIPE_SWEEP_RETRIES:-2}}"
|
|
|
|
{prepare_config}vllm bench sweep serve \
|
|
--serve-cmd "vllm serve --config '${{CONFIG_PATH}}'" \
|
|
--bench-cmd "{bench_cmd}" \
|
|
--serve-params "${{SCRIPT_DIR}}/{serve_params_name}" \
|
|
--bench-params "${{SCRIPT_DIR}}/bench_params.json" \
|
|
--output-dir "${{SCRIPT_DIR}}/results" \
|
|
--experiment-name {experiment_name} \
|
|
--warmup-num-prompts {workload.concurrency} \
|
|
${{CONTINUE_ON_ERROR_ARG}} \
|
|
"$@" || {{
|
|
if [[ "${{SWEEP_RETRY_COUNT}}" -ge "${{MAX_SWEEP_RETRIES}}" ]]; then
|
|
echo "Sweep failed after ${{MAX_SWEEP_RETRIES}} automatic retries." >&2
|
|
exit 1
|
|
fi
|
|
|
|
NEXT_RETRY=$((SWEEP_RETRY_COUNT + 1))
|
|
RETRY_ARGS=()
|
|
for arg in "$@"; do
|
|
[[ "$arg" == "--resume" ]] && continue
|
|
RETRY_ARGS+=("$arg")
|
|
done
|
|
|
|
EXPERIMENT_DIR="${{SCRIPT_DIR}}/results/{experiment_name}"
|
|
if [[ -d "${{EXPERIMENT_DIR}}" ]]; then
|
|
echo "Sweep command failed;" \
|
|
"retry ${{NEXT_RETRY}}/${{MAX_SWEEP_RETRIES}} with --resume"
|
|
RETRY_ARGS=(--resume "${{RETRY_ARGS[@]}}")
|
|
else
|
|
echo "Sweep command failed before resumable state was created;" \
|
|
"retry ${{NEXT_RETRY}}/${{MAX_SWEEP_RETRIES}} from the start"
|
|
fi
|
|
|
|
VLLM_RECIPE_SWEEP_RETRY_COUNT="${{NEXT_RETRY}}" exec "$0" "${{RETRY_ARGS[@]}}"
|
|
}}
|
|
"""
|
|
path.write_text(script, encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_recommend_script(
|
|
path: Path,
|
|
*,
|
|
config_rel: str,
|
|
env_rel: str,
|
|
workload: WorkloadHints,
|
|
results_dir: str = "results/runtime-tuning",
|
|
output_config: str = "recommended-config.yml",
|
|
output_json: str = "recommendation.json",
|
|
) -> None:
|
|
template_path = Path(__file__).with_name("sweep_recommendation.py")
|
|
source = template_path.read_text(encoding="utf-8")
|
|
|
|
replacements = {
|
|
"DEFAULT_CONFIG_PATH: str | None = None": (
|
|
f"DEFAULT_CONFIG_PATH: str | None = {config_rel!r}"
|
|
),
|
|
"DEFAULT_ENV_PATH: str | None = None": (
|
|
f"DEFAULT_ENV_PATH: str | None = {env_rel!r}"
|
|
),
|
|
"DEFAULT_TTFT_SLA_MS: float | None = None": (
|
|
f"DEFAULT_TTFT_SLA_MS: float | None = {workload.ttft_sla_ms!r}"
|
|
),
|
|
"DEFAULT_TPOT_SLA_MS: float | None = None": (
|
|
f"DEFAULT_TPOT_SLA_MS: float | None = {workload.tpot_sla_ms!r}"
|
|
),
|
|
'DEFAULT_RESULTS_DIR = "results/runtime-tuning"': (
|
|
f"DEFAULT_RESULTS_DIR = {results_dir!r}"
|
|
),
|
|
'DEFAULT_OUTPUT_CONFIG = "recommended-config.yml"': (
|
|
f"DEFAULT_OUTPUT_CONFIG = {output_config!r}"
|
|
),
|
|
'DEFAULT_OUTPUT_JSON = "recommendation.json"': (
|
|
f"DEFAULT_OUTPUT_JSON = {output_json!r}"
|
|
),
|
|
}
|
|
for marker, replacement in replacements.items():
|
|
if marker not in source:
|
|
raise ValueError(f"Recommender template marker not found: {marker}")
|
|
source = source.replace(marker, replacement, 1)
|
|
|
|
path.write_text(source, encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_guide(path: Path, workload: WorkloadHints) -> None:
|
|
sla_lines: list[str] = []
|
|
if workload.ttft_sla_ms is not None:
|
|
sla_lines.append(f"- TTFT objective: `{workload.ttft_sla_ms:g} ms`")
|
|
if workload.tpot_sla_ms is not None:
|
|
sla_lines.append(f"- TPOT objective: `{workload.tpot_sla_ms:g} ms`")
|
|
|
|
sla_text = "\n".join(sla_lines)
|
|
if sla_text:
|
|
sla_text = (
|
|
"\n## Supplied latency objectives\n\n"
|
|
+ sla_text
|
|
+ "\n\nThe generated benchmark uses these values with vLLM "
|
|
"`--goodput`.\n"
|
|
)
|
|
|
|
content = f"""# Optional Runtime Tuning Sweep
|
|
|
|
The generated `config.yml` is the **single initial suggestion** and can be
|
|
deployed directly. The scheduler sweep keeps explicit `max-num-seqs` at the
|
|
per-replica concurrency lower bound while benchmarking nearby values for:
|
|
|
|
- `max-num-batched-tokens`
|
|
|
|
It does not test explicit `max-num-seqs` values below per-replica concurrency,
|
|
because those values can introduce scheduler queueing and severe TTFT
|
|
degradation. The vLLM-default references still test whether an explicit
|
|
`max-num-seqs` override is needed.
|
|
|
|
In addition to nearby tuned values, the sweep includes three vLLM-default
|
|
references:
|
|
|
|
- vLLM default `max-num-seqs` with the initial batch-token budget
|
|
- initial `max-num-seqs` with vLLM default `max-num-batched-tokens`
|
|
- vLLM defaults for both parameters
|
|
|
|
The sweep runs every server from a generated `sweep_config.yml` that copies the
|
|
initial `config.yml` but removes these two scheduler keys. Tuned candidates add
|
|
explicit CLI values; a default-reference candidate simply omits the selected
|
|
key, allowing `vllm serve` to resolve its normal runtime default. This avoids
|
|
passing the literal string `None` and keeps defaults platform-, world-size-,
|
|
model-, and usage-context-aware.
|
|
|
|
## Run
|
|
|
|
```bash
|
|
./run_sweep.sh --dry-run
|
|
./run_sweep.sh
|
|
./recommend.py
|
|
```
|
|
|
|
For a quick one-run experiment:
|
|
|
|
```bash
|
|
rm -rf results/runtime-tuning
|
|
./run_sweep.sh --num-runs 1
|
|
./recommend.py
|
|
```
|
|
|
|
Resume an interrupted sweep:
|
|
|
|
```bash
|
|
./run_sweep.sh --resume
|
|
```
|
|
|
|
Before measurement, the generated script runs one unmeasured warmup containing
|
|
one full concurrency window ({workload.concurrency} prompts). It is saved as
|
|
`warmup.json` for auditing but excluded from recommendation statistics. By
|
|
default, vLLM then benchmarks each parameter combination three times.
|
|
|
|
The benchmark request count scales with the supplied workload concurrency using
|
|
the same model as vLLM's `PROMPTS_PER_CONCURRENCY` performance-benchmark
|
|
control:
|
|
|
|
```text
|
|
num_prompts = min(1000, max(100, concurrency * 10))
|
|
```
|
|
|
|
This targets about ten concurrency turnovers when neither bound applies, keeps
|
|
at least 100 requests for percentile/compliance measurements, and caps each run
|
|
at 1000 requests. All scheduler candidates use the same request count.
|
|
{sla_text}
|
|
## Outputs
|
|
|
|
`recommend.py` writes:
|
|
|
|
```text
|
|
recommended-config.yml
|
|
recommendation.json
|
|
```
|
|
|
|
With TTFT/TPOT objectives it requires duration-weighted combined compliance of
|
|
at least 99% plus median P99 TTFT/TPOT compliance. Scheduler candidates within
|
|
1% of the highest eligible output-token throughput are treated as practically
|
|
equivalent, and the recommender prefers fewer explicit scheduler overrides
|
|
within that set. Use `--throughput-equivalence-percent 0` for exact
|
|
highest-throughput selection. If no candidate qualifies, it records a
|
|
best-effort candidate but does not write `recommended-config.yml`. Without
|
|
latency objectives it applies the same scheduler equivalence policy.
|
|
Configurations with failed requests are excluded. TP/DP comparisons continue
|
|
to use exact highest-throughput selection.
|
|
|
|
`recommended-config.yml` copies the initial configuration and changes only
|
|
`max-num-seqs` and `max-num-batched-tokens`. If a vLLM-default reference wins,
|
|
the corresponding key is removed from `recommended-config.yml`, allowing
|
|
`vllm serve` to resolve that parameter from its normal runtime default policy.
|
|
"""
|
|
path.write_text(content, encoding="utf-8")
|
|
|
|
|
|
def _write_report_script(path: Path, *, workload: WorkloadHints) -> None:
|
|
template_path = Path(__file__).with_name("sweep_html_report.py")
|
|
source = template_path.read_text(encoding="utf-8")
|
|
replacements = {
|
|
"DEFAULT_TTFT_SLA_MS: float | None = None": (
|
|
f"DEFAULT_TTFT_SLA_MS: float | None = {workload.ttft_sla_ms!r}"
|
|
),
|
|
"DEFAULT_TPOT_SLA_MS: float | None = None": (
|
|
f"DEFAULT_TPOT_SLA_MS: float | None = {workload.tpot_sla_ms!r}"
|
|
),
|
|
}
|
|
for marker, replacement in replacements.items():
|
|
if marker not in source:
|
|
raise ValueError(f"HTML report template marker not found: {marker}")
|
|
source = source.replace(marker, replacement, 1)
|
|
path.write_text(source, encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_post_benchmark_analysis_files(
|
|
directory: Path, *, workload: WorkloadHints
|
|
) -> list[Path]:
|
|
"""Copy standalone reporting helpers into a generated sweep package."""
|
|
source_dir = Path(__file__).resolve().parent
|
|
generated: list[Path] = []
|
|
for source_name, target_name, executable in (
|
|
("sweep_visualization.py", "visualize.py", True),
|
|
("requirements.txt", "requirements.txt", False),
|
|
("VISUALIZATION.md", "VISUALIZATION.md", False),
|
|
):
|
|
source = (
|
|
source_dir / source_name
|
|
if source_name.endswith(".py")
|
|
else source_dir.parent / source_name
|
|
)
|
|
target = directory / target_name
|
|
target.write_text(source.read_text(encoding="utf-8"), encoding="utf-8")
|
|
if executable:
|
|
target.chmod(target.stat().st_mode | 0o111)
|
|
generated.append(target)
|
|
report = directory / "report.py"
|
|
_write_report_script(report, workload=workload)
|
|
generated.append(report)
|
|
return generated
|
|
|
|
|
|
def write_sweep_files(
|
|
output_dir: str,
|
|
*,
|
|
config_path: str,
|
|
env_path: str,
|
|
config: dict[str, Any],
|
|
workload: WorkloadHints,
|
|
) -> list[Path]:
|
|
"""Write an optional sweep package around the initial config suggestion."""
|
|
validate_sweep_workload(workload)
|
|
|
|
directory = Path(output_dir)
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
|
|
sweep_config = directory / "sweep_config.yml"
|
|
serve_params = directory / "serve_params.json"
|
|
bench_params = directory / "bench_params.json"
|
|
run_script = directory / "run_sweep.sh"
|
|
recommend_script = directory / "recommend.py"
|
|
guide = directory / "SWEEP.md"
|
|
|
|
config_rel = _relative_to(directory, config_path)
|
|
sweep_config_rel = _relative_to(directory, str(sweep_config))
|
|
env_rel = _relative_to(directory, env_path)
|
|
|
|
_write_sweep_config(sweep_config, config)
|
|
_write_json(serve_params, build_serve_params(config))
|
|
_write_json(bench_params, build_bench_params(workload))
|
|
request_model, tokenizer = _benchmark_models(config)
|
|
_write_run_script(
|
|
run_script,
|
|
config_rel=sweep_config_rel,
|
|
env_rel=env_rel,
|
|
request_model=request_model,
|
|
tokenizer=tokenizer,
|
|
workload=workload,
|
|
)
|
|
_write_recommend_script(
|
|
recommend_script,
|
|
config_rel=config_rel,
|
|
env_rel=env_rel,
|
|
workload=workload,
|
|
)
|
|
_write_guide(guide, workload)
|
|
analysis_files = _write_post_benchmark_analysis_files(directory, workload=workload)
|
|
|
|
return [
|
|
sweep_config,
|
|
serve_params,
|
|
bench_params,
|
|
run_script,
|
|
recommend_script,
|
|
guide,
|
|
*analysis_files,
|
|
]
|
|
|
|
|
|
def write_parallel_layout_sweep_files(
|
|
output_dir: str,
|
|
*,
|
|
config_path: str,
|
|
env_path: str,
|
|
config: dict[str, Any],
|
|
workload: WorkloadHints,
|
|
numa_node_count: int,
|
|
tune_scheduler: bool = False,
|
|
) -> list[Path]:
|
|
"""Write a standalone NUMA-aware TP/DP sweep package."""
|
|
validate_sweep_workload(workload)
|
|
directory = Path(output_dir)
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
|
|
parallel_params = directory / "parallel_layout_serve_params.json"
|
|
parallel_skips = directory / "parallel_layout_skips.json"
|
|
parallel_config = directory / "parallel-layout-benchmark-config.yml"
|
|
bench_params = directory / "bench_params.json"
|
|
run_parallel = directory / "run_parallel_layout_sweep.sh"
|
|
recommend_parallel = directory / "recommend_parallel_layout.py"
|
|
guide = directory / "SWEEP.md"
|
|
|
|
initial_config_rel = _relative_to(directory, config_path)
|
|
parallel_config_rel = _relative_to(directory, str(parallel_config))
|
|
selected_config_rel = "parallel-layout-config.yml"
|
|
env_rel = _relative_to(directory, env_path)
|
|
request_model, tokenizer = _benchmark_models(config)
|
|
|
|
_write_sweep_config(parallel_config, config)
|
|
benchmark_config_rel = initial_config_rel if tune_scheduler else parallel_config_rel
|
|
|
|
try:
|
|
model_metadata = inspect_model_parallel_metadata(config)
|
|
except Exception as exc:
|
|
print(
|
|
"Warning: could not inspect model parallel metadata; retaining the "
|
|
f"generic layouts and relying on sweep error isolation: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
model_metadata = UNKNOWN_MODEL_PARALLEL_METADATA
|
|
cpu_moe_dp_supported = (
|
|
cpu_supports_moe_dp_collectives() if model_metadata.is_moe else True
|
|
)
|
|
parallel_candidates, skipped_candidates = build_parallel_layout_plan(
|
|
config,
|
|
workload,
|
|
numa_node_count,
|
|
model_metadata=model_metadata,
|
|
cpu_moe_dp_supported=cpu_moe_dp_supported,
|
|
tune_scheduler=tune_scheduler,
|
|
)
|
|
_write_json(parallel_params, parallel_candidates)
|
|
_write_json(parallel_skips, skipped_candidates)
|
|
_write_json(bench_params, build_bench_params(workload))
|
|
_write_run_script(
|
|
run_parallel,
|
|
config_rel=benchmark_config_rel,
|
|
env_rel=env_rel,
|
|
request_model=request_model,
|
|
tokenizer=tokenizer,
|
|
workload=workload,
|
|
serve_params_name=parallel_params.name,
|
|
experiment_name="parallel-layout",
|
|
)
|
|
_write_recommend_script(
|
|
recommend_parallel,
|
|
config_rel=initial_config_rel,
|
|
env_rel=env_rel,
|
|
workload=workload,
|
|
results_dir="results/parallel-layout",
|
|
output_config=selected_config_rel,
|
|
output_json="parallel-layout-recommendation.json",
|
|
)
|
|
|
|
guide.write_text(
|
|
f"""# Parallel-Layout Sweep
|
|
|
|
This package tunes only the NUMA-aware TP/DP layout.
|
|
|
|
Primary candidates use every effective NUMA node:
|
|
|
|
```text
|
|
tensor_parallel_size * data_parallel_size = {numa_node_count}
|
|
```
|
|
|
|
Tensor parallelism is restricted to `1`, `2`, `4`, or `8`. The largest
|
|
supported TP not exceeding the NUMA-node count is also included, even when it
|
|
leaves some NUMA nodes idle.
|
|
|
|
Before benchmarking, the generator applies two MoE safety checks:
|
|
|
|
- When the CPU communicator lacks variable-size MoE collectives, MoE candidates
|
|
use `DP=1`; every supported TP size is retained as a replacement candidate.
|
|
- With expert parallelism disabled and a known MoE intermediate size, candidates
|
|
must satisfy `moe_intermediate_size % (TP * DP * PCP) == 0`.
|
|
|
|
Rejected layouts and their reasons are written to
|
|
`parallel_layout_skips.json`. If model metadata cannot be inspected, generic
|
|
layouts are retained and the runner's per-combination error isolation applies.
|
|
|
|
Run:
|
|
|
|
```bash
|
|
./run_parallel_layout_sweep.sh --dry-run
|
|
./run_parallel_layout_sweep.sh
|
|
./recommend_parallel_layout.py
|
|
```
|
|
|
|
The recommender writes:
|
|
|
|
```text
|
|
parallel-layout-config.yml
|
|
parallel-layout-recommendation.json
|
|
```
|
|
|
|
This mode stops after TP/DP selection. Use `--generate-full-sweep` when
|
|
concurrency and scheduler tuning should follow automatically.
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
analysis_files = _write_post_benchmark_analysis_files(directory, workload=workload)
|
|
|
|
return [
|
|
parallel_params,
|
|
parallel_skips,
|
|
bench_params,
|
|
run_parallel,
|
|
recommend_parallel,
|
|
guide,
|
|
*analysis_files,
|
|
]
|
|
|
|
|
|
def build_concurrency_bench_params(workload: WorkloadHints) -> list[dict[str, Any]]:
|
|
"""Build the fixed workload shape for concurrency tuning."""
|
|
validate_sweep_workload(workload)
|
|
assert workload.input_tokens is not None
|
|
assert workload.output_tokens is not None
|
|
assert workload.concurrency is not None
|
|
return [
|
|
{
|
|
"_benchmark_name": "user_workload",
|
|
"random_input_len": workload.input_tokens,
|
|
"random_output_len": workload.output_tokens,
|
|
"num_prompts": _num_prompts_for_concurrency(workload.concurrency),
|
|
}
|
|
]
|
|
|
|
|
|
def _write_adaptive_concurrency_script(path: Path) -> None:
|
|
template_path = Path(__file__).with_name("adaptive_concurrency.py")
|
|
path.write_text(template_path.read_text(encoding="utf-8"), encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_concurrency_run_script(
|
|
path: Path,
|
|
*,
|
|
config_rel: str,
|
|
env_rel: str,
|
|
request_model: str,
|
|
tokenizer: str,
|
|
workload: WorkloadHints,
|
|
) -> None:
|
|
bench_parts = [
|
|
"vllm bench serve",
|
|
"--backend vllm",
|
|
f"--model {shlex.quote(request_model)}",
|
|
f"--tokenizer {shlex.quote(tokenizer)}",
|
|
"--dataset-name random",
|
|
"--request-rate inf",
|
|
"--ignore-eos",
|
|
"--metric-percentiles 99",
|
|
]
|
|
goodput_pairs: list[str] = []
|
|
if workload.ttft_sla_ms is not None:
|
|
goodput_pairs.append(f"ttft:{workload.ttft_sla_ms:g}")
|
|
if workload.tpot_sla_ms is not None:
|
|
goodput_pairs.append(f"tpot:{workload.tpot_sla_ms:g}")
|
|
if goodput_pairs:
|
|
bench_parts.append(
|
|
"--goodput " + " ".join(shlex.quote(pair) for pair in goodput_pairs)
|
|
)
|
|
|
|
bench_cmd = " ".join(bench_parts)
|
|
use_adaptive_sla_search = bool(
|
|
workload.ttft_sla_ms is not None or workload.tpot_sla_ms is not None
|
|
)
|
|
adaptive_args = [
|
|
f"--seed-concurrency {workload.concurrency}",
|
|
f"--input-tokens {workload.input_tokens}",
|
|
f"--output-tokens {workload.output_tokens}",
|
|
]
|
|
if workload.ttft_sla_ms is not None:
|
|
adaptive_args.append(f"--ttft-sla-ms {workload.ttft_sla_ms:g}")
|
|
if workload.tpot_sla_ms is not None:
|
|
adaptive_args.append(f"--tpot-sla-ms {workload.tpot_sla_ms:g}")
|
|
adaptive_args_text = " \\\n ".join(adaptive_args)
|
|
|
|
script = f"""#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${{BASH_SOURCE[0]}}")" && pwd)"
|
|
CONFIG_PATH="${{SCRIPT_DIR}}/{config_rel}"
|
|
ENV_PATH="${{SCRIPT_DIR}}/{env_rel}"
|
|
|
|
source "${{ENV_PATH}}"
|
|
|
|
# Released/nightly vLLM images keep recipe-relative assets such as
|
|
# examples/*.jinja under /vllm-workspace. Launch sweep server subprocesses
|
|
# from there when available so relative recipe assets resolve like they do
|
|
# in serve_with_recipe.sh.
|
|
if [[ -d /vllm-workspace/examples ]]; then
|
|
cd /vllm-workspace
|
|
fi
|
|
|
|
# recipe tools may be newer than the installed vLLM package. Only pass the
|
|
# resilience flag when this vLLM CLI actually supports it.
|
|
CONTINUE_ON_ERROR_ARG=""
|
|
if [[ "{int(use_adaptive_sla_search)}" == "1" ]] || \
|
|
vllm bench sweep serve_workload --help 2>&1 | grep -q -- "--continue-on-error"; then
|
|
CONTINUE_ON_ERROR_ARG="--continue-on-error"
|
|
fi
|
|
|
|
# On older vLLM versions, a failed benchmark aborts the sweep process. Retry the
|
|
# interrupted sweep with --resume so one transient run failure does not discard
|
|
# completed work or force a manual restart.
|
|
SWEEP_RETRY_COUNT="${{VLLM_RECIPE_SWEEP_RETRY_COUNT:-0}}"
|
|
MAX_SWEEP_RETRIES="${{VLLM_RECIPE_SWEEP_RETRIES:-2}}"
|
|
SERVER_READY_TIMEOUT="${{VLLM_RECIPE_SERVER_READY_TIMEOUT:-1800}}"
|
|
|
|
run_concurrency_sweep() {{
|
|
if [[ "{int(use_adaptive_sla_search)}" == "1" ]]; then
|
|
python3 "${{SCRIPT_DIR}}/adaptive_concurrency.py" \
|
|
--serve-cmd "vllm serve --config '${{CONFIG_PATH}}'" \
|
|
--bench-cmd "{bench_cmd}" \
|
|
--results-dir "${{SCRIPT_DIR}}/results/concurrency-tuning" \
|
|
{adaptive_args_text} \
|
|
--server-ready-timeout "${{SERVER_READY_TIMEOUT}}" \
|
|
${{CONTINUE_ON_ERROR_ARG}} \
|
|
"$@"
|
|
else
|
|
vllm bench sweep serve_workload \
|
|
--serve-cmd "vllm serve --config '${{CONFIG_PATH}}'" \
|
|
--bench-cmd "{bench_cmd}" \
|
|
--bench-params "${{SCRIPT_DIR}}/concurrency_bench_params.json" \
|
|
--output-dir "${{SCRIPT_DIR}}/results" \
|
|
--experiment-name concurrency-tuning \
|
|
--workload-var max_concurrency \
|
|
--workload-iters 10 \
|
|
--warmup-num-prompts {workload.concurrency} \
|
|
${{CONTINUE_ON_ERROR_ARG}} \
|
|
"$@"
|
|
fi
|
|
}}
|
|
|
|
run_concurrency_sweep || {{
|
|
if [[ "${{SWEEP_RETRY_COUNT}}" -ge "${{MAX_SWEEP_RETRIES}}" ]]; then
|
|
echo "Sweep failed after ${{MAX_SWEEP_RETRIES}} automatic retries." >&2
|
|
exit 1
|
|
fi
|
|
|
|
NEXT_RETRY=$((SWEEP_RETRY_COUNT + 1))
|
|
RETRY_ARGS=()
|
|
for arg in "$@"; do
|
|
[[ "$arg" == "--resume" ]] && continue
|
|
RETRY_ARGS+=("$arg")
|
|
done
|
|
|
|
EXPERIMENT_DIR="${{SCRIPT_DIR}}/results/concurrency-tuning"
|
|
if [[ -d "${{EXPERIMENT_DIR}}" ]]; then
|
|
echo "Sweep command failed;" \
|
|
"retry ${{NEXT_RETRY}}/${{MAX_SWEEP_RETRIES}} with --resume"
|
|
RETRY_ARGS=(--resume "${{RETRY_ARGS[@]}}")
|
|
else
|
|
echo "Sweep command failed before resumable state was created;" \
|
|
"retry ${{NEXT_RETRY}}/${{MAX_SWEEP_RETRIES}} from the start"
|
|
fi
|
|
|
|
VLLM_RECIPE_SWEEP_RETRY_COUNT="${{NEXT_RETRY}}" exec "$0" "${{RETRY_ARGS[@]}}"
|
|
}}
|
|
"""
|
|
path.write_text(script, encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_concurrency_recommend_script(path: Path, *, workload: WorkloadHints) -> None:
|
|
template_path = Path(__file__).with_name("concurrency_recommendation.py")
|
|
source = template_path.read_text(encoding="utf-8")
|
|
replacements = {
|
|
"DEFAULT_TTFT_SLA_MS: float | None = None": (
|
|
f"DEFAULT_TTFT_SLA_MS: float | None = {workload.ttft_sla_ms!r}"
|
|
),
|
|
"DEFAULT_TPOT_SLA_MS: float | None = None": (
|
|
f"DEFAULT_TPOT_SLA_MS: float | None = {workload.tpot_sla_ms!r}"
|
|
),
|
|
"DEFAULT_SEED_CONCURRENCY: int | None = None": (
|
|
f"DEFAULT_SEED_CONCURRENCY: int | None = {workload.concurrency!r}"
|
|
),
|
|
}
|
|
for marker, replacement in replacements.items():
|
|
if marker not in source:
|
|
raise ValueError(f"Concurrency recommender marker not found: {marker}")
|
|
source = source.replace(marker, replacement, 1)
|
|
path.write_text(source, encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_scheduler_prepare_script(path: Path) -> None:
|
|
template_path = Path(__file__).with_name("scheduler_sweep_prepare.py")
|
|
path.write_text(template_path.read_text(encoding="utf-8"), encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_scheduler_wrapper(path: Path, *, workload: WorkloadHints) -> None:
|
|
assert workload.input_tokens is not None
|
|
assert workload.output_tokens is not None
|
|
args = [
|
|
'python3 "${SCRIPT_DIR}/prepare_scheduler.py"',
|
|
'--config "${SCRIPT_DIR}/parallel-layout-config.yml"',
|
|
'--concurrency-recommendation "${SCRIPT_DIR}/concurrency-recommendation.json"',
|
|
f"--input-tokens {workload.input_tokens}",
|
|
f"--output-tokens {workload.output_tokens}",
|
|
]
|
|
if workload.tpot_sla_ms is not None:
|
|
args.append(f"--tpot-sla-ms {workload.tpot_sla_ms:g}")
|
|
if workload.target_qps is not None:
|
|
args.append(f"--target-qps {workload.target_qps:g}")
|
|
args.extend(
|
|
[
|
|
'--output-config "${SCRIPT_DIR}/parallel-layout-sweep-config.yml"',
|
|
'--serve-params "${SCRIPT_DIR}/serve_params.json"',
|
|
'--bench-params "${SCRIPT_DIR}/bench_params.json"',
|
|
]
|
|
)
|
|
prepare_cmd = " \\\n ".join(args)
|
|
script = f"""#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${{BASH_SOURCE[0]}}")" && pwd)"
|
|
|
|
{prepare_cmd}
|
|
|
|
exec "${{SCRIPT_DIR}}/run_sweep.sh" "$@"
|
|
"""
|
|
path.write_text(script, encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _recommend_sla_args(workload: WorkloadHints) -> list[str]:
|
|
args = []
|
|
if workload.ttft_sla_ms is not None:
|
|
args.append(f"--ttft-sla-ms {workload.ttft_sla_ms:g}")
|
|
if workload.tpot_sla_ms is not None:
|
|
args.append(f"--tpot-sla-ms {workload.tpot_sla_ms:g}")
|
|
return args
|
|
|
|
|
|
def _write_full_run_script(path: Path, *, workload: WorkloadHints) -> None:
|
|
recommend_sla_args = " ".join(_recommend_sla_args(workload))
|
|
script = f"""#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${{BASH_SOURCE[0]}}")" && pwd)"
|
|
STAGES=all
|
|
RUN_ARGS=()
|
|
SERVER_READY_TIMEOUT="${{VLLM_RECIPE_SERVER_READY_TIMEOUT:-1800}}"
|
|
RECOMMEND_SLA_ARGS=({recommend_sla_args})
|
|
while (($#)); do
|
|
case "$1" in
|
|
--stages)
|
|
if (($# < 2)); then
|
|
echo "--stages requires a comma-separated stage list." >&2
|
|
exit 2
|
|
fi
|
|
STAGES="$2"
|
|
shift 2
|
|
;;
|
|
--stages=*) STAGES="${{1#*=}}"; shift ;;
|
|
--server-ready-timeout)
|
|
if (($# < 2)); then
|
|
echo "--server-ready-timeout requires a value." >&2
|
|
exit 2
|
|
fi
|
|
SERVER_READY_TIMEOUT="$2"
|
|
RUN_ARGS+=("$1" "$2")
|
|
shift 2
|
|
;;
|
|
--server-ready-timeout=*)
|
|
SERVER_READY_TIMEOUT="${{1#*=}}"
|
|
RUN_ARGS+=("$1")
|
|
shift
|
|
;;
|
|
--help|-h)
|
|
echo "Usage: $0 [--stages all|parallel-layout,concurrency,scheduler] [sweep args]"
|
|
echo "Selected stages always run in dependency order. Default: all."
|
|
exit 0
|
|
;;
|
|
*) RUN_ARGS+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
case "$STAGES" in
|
|
all) STAGES=parallel-layout,concurrency,scheduler ;;
|
|
""|,*|*,|*,,*) echo "Invalid stage list: $STAGES" >&2; exit 2 ;;
|
|
esac
|
|
IFS=, read -r -a SELECTED_STAGES <<< "$STAGES"
|
|
for stage in "${{SELECTED_STAGES[@]}}"; do
|
|
case "$stage" in
|
|
parallel-layout|concurrency|scheduler) ;;
|
|
*) echo "Unknown sweep stage: $stage" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
has_stage() {{ [[ ",$STAGES," == *",$1,"* ]]; }}
|
|
|
|
# Keep the concurrency runner's server-start timeout explicit even when sweep
|
|
# arguments pass through multiple generated wrapper scripts.
|
|
export VLLM_RECIPE_SERVER_READY_TIMEOUT="${{SERVER_READY_TIMEOUT}}"
|
|
|
|
# Check dependencies before starting any benchmarks.
|
|
if has_stage scheduler && ! has_stage concurrency &&
|
|
[[ ! -f "${{SCRIPT_DIR}}/concurrency-recommendation.json" ]]; then
|
|
echo "Scheduler requires concurrency-recommendation.json; run concurrency first." >&2
|
|
exit 2
|
|
fi
|
|
if ! has_stage parallel-layout &&
|
|
[[ ! -f "${{SCRIPT_DIR}}/parallel-layout-config.yml" ]]; then
|
|
cp "${{SCRIPT_DIR}}/initial-config.yml" "${{SCRIPT_DIR}}/parallel-layout-config.yml"
|
|
fi
|
|
|
|
if has_stage parallel-layout; then
|
|
"${{SCRIPT_DIR}}/run_parallel_layout_sweep.sh" "${{RUN_ARGS[@]}}"
|
|
"${{SCRIPT_DIR}}/recommend_parallel_layout.py" "${{RECOMMEND_SLA_ARGS[@]}}"
|
|
fi
|
|
if has_stage concurrency; then
|
|
"${{SCRIPT_DIR}}/run_concurrency_sweep.sh" "${{RUN_ARGS[@]}}"
|
|
"${{SCRIPT_DIR}}/recommend_concurrency.py" "${{RECOMMEND_SLA_ARGS[@]}}"
|
|
fi
|
|
if has_stage scheduler; then
|
|
"${{SCRIPT_DIR}}/run_scheduler_sweep.sh" "${{RUN_ARGS[@]}}"
|
|
"${{SCRIPT_DIR}}/recommend.py" "${{RECOMMEND_SLA_ARGS[@]}}"
|
|
fi
|
|
|
|
if ! "${{SCRIPT_DIR}}/report.py" "${{RECOMMEND_SLA_ARGS[@]}}"; then
|
|
echo "Warning: sweep completed, but HTML report generation failed." >&2
|
|
fi
|
|
"""
|
|
path.write_text(script, encoding="utf-8")
|
|
path.chmod(path.stat().st_mode | 0o111)
|
|
|
|
|
|
def _write_full_guide(path: Path, workload: WorkloadHints) -> None:
|
|
content = f"""# End-to-End Runtime Tuning
|
|
|
|
This package keeps one fixed workload shape and tunes in dependency order:
|
|
|
|
```text
|
|
TP/DP -> max_concurrency -> scheduler
|
|
```
|
|
|
|
The supplied `--concurrency={workload.concurrency}` is the representative load
|
|
used for TP/DP selection and to size the Workload Explorer dataset. Stage 2
|
|
selects the measured SLA-feasible `max_concurrency`. Stage 3 recalculates its
|
|
scheduler baseline from that concurrency and the selected DP layout.
|
|
|
|
Run all stages:
|
|
|
|
```bash
|
|
./run_full_sweep.sh
|
|
```
|
|
|
|
After the final recommendation, the full runner writes `sweep-report.html`.
|
|
Report-generation failures are warnings and do not change sweep success.
|
|
|
|
Run selected stages (always executed in dependency order):
|
|
|
|
```bash
|
|
./run_full_sweep.sh --stages concurrency
|
|
./run_full_sweep.sh --stages concurrency,scheduler
|
|
./run_full_sweep.sh --stages scheduler --resume
|
|
```
|
|
|
|
When parallel-layout tuning is skipped, an existing `parallel-layout-config.yml`
|
|
is reused; otherwise `initial-config.yml` supplies the fixed TP/DP layout.
|
|
Scheduler-only runs require an existing `concurrency-recommendation.json`.
|
|
Other arguments are forwarded to the selected benchmark runners.
|
|
|
|
Or run each stage explicitly:
|
|
|
|
```bash
|
|
./run_parallel_layout_sweep.sh
|
|
./recommend_parallel_layout.py
|
|
./run_concurrency_sweep.sh
|
|
./recommend_concurrency.py
|
|
./run_scheduler_sweep.sh
|
|
./recommend.py
|
|
```
|
|
|
|
Generated recipe sweeps use `--continue-on-error`. An isolated warmup or
|
|
benchmark invocation failure is recorded and skipped. A combination with no
|
|
successful measured runs remains retryable with `--resume`.
|
|
"""
|
|
path.write_text(content, encoding="utf-8")
|
|
|
|
|
|
def write_concurrency_sweep_files(
|
|
output_dir: str,
|
|
*,
|
|
config_path: str,
|
|
env_path: str,
|
|
config: dict[str, Any],
|
|
workload: WorkloadHints,
|
|
tune_scheduler: bool = False,
|
|
) -> list[Path]:
|
|
"""Write a standalone max_concurrency tuning package."""
|
|
validate_sweep_workload(workload)
|
|
directory = Path(output_dir)
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
|
|
bench_params = directory / "concurrency_bench_params.json"
|
|
concurrency_config = directory / "concurrency-benchmark-config.yml"
|
|
adaptive_script = directory / "adaptive_concurrency.py"
|
|
run_script = directory / "run_concurrency_sweep.sh"
|
|
recommend_script = directory / "recommend_concurrency.py"
|
|
|
|
config_rel = _relative_to(directory, config_path)
|
|
concurrency_config_rel = _relative_to(directory, str(concurrency_config))
|
|
env_rel = _relative_to(directory, env_path)
|
|
request_model, tokenizer = _benchmark_models(config)
|
|
|
|
_write_sweep_config(concurrency_config, config)
|
|
benchmark_config_rel = config_rel if tune_scheduler else concurrency_config_rel
|
|
|
|
_write_json(bench_params, build_concurrency_bench_params(workload))
|
|
_write_adaptive_concurrency_script(adaptive_script)
|
|
_write_concurrency_run_script(
|
|
run_script,
|
|
config_rel=benchmark_config_rel,
|
|
env_rel=env_rel,
|
|
request_model=request_model,
|
|
tokenizer=tokenizer,
|
|
workload=workload,
|
|
)
|
|
_write_concurrency_recommend_script(recommend_script, workload=workload)
|
|
analysis_files = _write_post_benchmark_analysis_files(directory, workload=workload)
|
|
return [
|
|
bench_params,
|
|
concurrency_config,
|
|
adaptive_script,
|
|
run_script,
|
|
recommend_script,
|
|
*analysis_files,
|
|
]
|
|
|
|
|
|
def write_full_sweep_files(
|
|
output_dir: str,
|
|
*,
|
|
config_path: str,
|
|
env_path: str,
|
|
config: dict[str, Any],
|
|
workload: WorkloadHints,
|
|
numa_node_count: int,
|
|
tune_scheduler: bool = False,
|
|
) -> list[Path]:
|
|
"""Write TP/DP -> concurrency -> scheduler end-to-end tuning artifacts."""
|
|
files = write_parallel_layout_sweep_files(
|
|
output_dir,
|
|
config_path=config_path,
|
|
env_path=env_path,
|
|
config=config,
|
|
workload=workload,
|
|
numa_node_count=numa_node_count,
|
|
tune_scheduler=tune_scheduler,
|
|
)
|
|
|
|
directory = Path(output_dir)
|
|
concurrency_bench = directory / "concurrency_bench_params.json"
|
|
adaptive_concurrency = directory / "adaptive_concurrency.py"
|
|
run_concurrency = directory / "run_concurrency_sweep.sh"
|
|
recommend_concurrency = directory / "recommend_concurrency.py"
|
|
prepare_scheduler = directory / "prepare_scheduler.py"
|
|
scheduler_runner = directory / "run_sweep.sh"
|
|
recommend_scheduler = directory / "recommend.py"
|
|
run_scheduler = directory / "run_scheduler_sweep.sh"
|
|
run_full = directory / "run_full_sweep.sh"
|
|
guide = directory / "SWEEP.md"
|
|
|
|
env_rel = _relative_to(directory, env_path)
|
|
request_model, tokenizer = _benchmark_models(config)
|
|
|
|
_write_json(concurrency_bench, build_concurrency_bench_params(workload))
|
|
_write_adaptive_concurrency_script(adaptive_concurrency)
|
|
_write_concurrency_run_script(
|
|
run_concurrency,
|
|
config_rel="parallel-layout-config.yml",
|
|
env_rel=env_rel,
|
|
request_model=request_model,
|
|
tokenizer=tokenizer,
|
|
workload=workload,
|
|
)
|
|
_write_concurrency_recommend_script(recommend_concurrency, workload=workload)
|
|
|
|
# Full tuning owns the scheduler stage. The standalone parallel-layout mode
|
|
# intentionally stops after TP/DP selection.
|
|
_write_scheduler_prepare_script(prepare_scheduler)
|
|
_write_run_script(
|
|
scheduler_runner,
|
|
config_rel="parallel-layout-sweep-config.yml",
|
|
env_rel=env_rel,
|
|
request_model=request_model,
|
|
tokenizer=tokenizer,
|
|
workload=workload,
|
|
)
|
|
_write_recommend_script(
|
|
recommend_scheduler,
|
|
config_rel="parallel-layout-config.yml",
|
|
env_rel=env_rel,
|
|
workload=workload,
|
|
)
|
|
_write_scheduler_wrapper(run_scheduler, workload=workload)
|
|
|
|
initial_config = directory / "initial-config.yml"
|
|
initial_config.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8")
|
|
_write_full_run_script(run_full, workload=workload)
|
|
_write_full_guide(guide, workload)
|
|
|
|
return [
|
|
*files,
|
|
concurrency_bench,
|
|
adaptive_concurrency,
|
|
run_concurrency,
|
|
recommend_concurrency,
|
|
prepare_scheduler,
|
|
scheduler_runner,
|
|
recommend_scheduler,
|
|
run_scheduler,
|
|
run_full,
|
|
initial_config,
|
|
]
|