[Bugfix] Redact credentials from benchmark logs (#56662)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

benchmarks/multi_turn/benchmark_serving_multi_turn.py

@@ -32,8 +32,11 @@ from bench_dataset import (
 from bench_utils import TEXT_SEPARATOR, Color, logger
 from transformers import AutoTokenizer  # type: ignore
 
+from vllm.benchmarks.lib.utils import redact_sensitive_namespace
+
 NUM_TOKENS_FROM_DATASET = 0
 TERM_SIGNAL = None
+_SENSITIVE_ARG_FIELDS = ("api_key", "header")
 
 
 class ConversationSampling(str, Enum):
@@ -1548,7 +1551,7 @@ async def main() -> None:
 
     args = parser.parse_args()
 
-    logger.info(args)
+    logger.info(redact_sensitive_namespace(args, _SENSITIVE_ARG_FIELDS))
 
     logger.info(f"{Color.GREEN}Input parameters:{Color.RESET}")
     logger.info(f"url={args.url}")

vllm/benchmarks/lib/utils.py

@@ -9,6 +9,17 @@ from contextlib import contextmanager
 from typing import Any
 
 
+def redact_sensitive_namespace(
+    args: argparse.Namespace, fields: tuple[str, ...]
+) -> argparse.Namespace:
+    """Return a copy of CLI arguments with selected values redacted."""
+    redacted_args = argparse.Namespace(**vars(args))
+    for field in fields:
+        if getattr(redacted_args, field, None) is not None:
+            setattr(redacted_args, field, "***")
+    return redacted_args
+
+
 def extract_field(
     args: argparse.Namespace, extra_info: dict[str, Any], field_name: str
 ) -> str:

vllm/benchmarks/serve.py

@@ -51,13 +51,18 @@ from vllm.benchmarks.lib.endpoint_request_func import (
     RequestFuncOutput,
 )
 from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
-from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
+from vllm.benchmarks.lib.utils import (
+    convert_to_pytorch_benchmark_format,
+    redact_sensitive_namespace,
+    write_to_json,
+)
 from vllm.tokenizers import TokenizerLike, get_tokenizer
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm.utils.gc_utils import freeze_gc_heap
 from vllm.utils.network_utils import join_host_port
 
 MILLISECONDS_TO_SECONDS_CONVERSION = 1000
+_SENSITIVE_ARG_FIELDS = ("header",)
 
 
 def _merge_overrides(base: dict | None, override: dict | None) -> dict | None:
@@ -1536,7 +1541,7 @@ def save_to_pytorch_benchmark_format(
     # later if needed
     ignored_metrics = ["ttfts", "itls", "generated_texts", "errors"]
     pt_records = convert_to_pytorch_benchmark_format(
-        args=args,
+        args=redact_sensitive_namespace(args, _SENSITIVE_ARG_FIELDS),
         metrics={k: [results[k]] for k in metrics if k in results},
         extra_info={
             k: results[k]
@@ -2013,7 +2018,7 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
 
 
 async def main_async(args: argparse.Namespace) -> dict[str, Any]:
-    print(args)
+    print(redact_sensitive_namespace(args, _SENSITIVE_ARG_FIELDS))
     if args.max_concurrency is not None and args.max_concurrency <= 0:
         raise ValueError("--max-concurrency must be greater than 0")