[Bugfix] Fix max_model_len="auto" handling (#31260)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-12-24 19:15:27 +08:00
committed by GitHub
parent bd89ce16d2
commit 7adeb4bfa8
3 changed files with 79 additions and 82 deletions
+26 -15
View File
@@ -297,16 +297,14 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
elif contains_type(type_hints, set):
kwargs[name].update(collection_to_kwargs(type_hints, set))
elif contains_type(type_hints, int):
kwargs[name]["type"] = int
# Special case for large integers
human_readable_ints = {
"max_model_len",
"max_num_batched_tokens",
"kv_cache_memory_bytes",
}
if name in human_readable_ints:
if name == "max_model_len":
kwargs[name]["type"] = human_readable_int_or_auto
kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
elif name in ("max_num_batched_tokens", "kv_cache_memory_bytes"):
kwargs[name]["type"] = human_readable_int
kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
else:
kwargs[name]["type"] = int
elif contains_type(type_hints, float):
kwargs[name]["type"] = float
elif contains_type(type_hints, dict) and (
@@ -2042,23 +2040,17 @@ def _raise_unsupported_error(feature_name: str):
raise NotImplementedError(msg)
def human_readable_int(value):
def human_readable_int(value: str) -> int:
"""Parse human-readable integers like '1k', '2M', etc.
Including decimal values with decimal multipliers.
Also accepts -1 or 'auto' as a special value for auto-detection.
Examples:
- '1k' -> 1,000
- '1K' -> 1,024
- '25.6k' -> 25,600
- '-1' or 'auto' -> -1 (special value for auto-detection)
"""
value = value.strip()
# Handle -1 or 'auto' as a special value for auto-detection
if value == "-1" or value.lower() == "auto":
return -1
match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value)
if match:
decimal_multiplier = {
@@ -2092,3 +2084,22 @@ def human_readable_int(value):
# Regular plain number.
return int(value)
def human_readable_int_or_auto(value: str) -> int:
"""Parse human-readable integers like '1k', '2M', etc.
Including decimal values with decimal multipliers.
Also accepts -1 or 'auto' as a special value for auto-detection.
Examples:
- '1k' -> 1,000
- '1K' -> 1,024
- '25.6k' -> 25,600
- '-1' or 'auto' -> -1 (special value for auto-detection)
"""
value = value.strip()
if value == "-1" or value.lower() == "auto":
return -1
return human_readable_int(value)