[Bugfix] Fix max_model_len="auto" handling (#31260)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+26
-15
@@ -297,16 +297,14 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
|
||||
elif contains_type(type_hints, set):
|
||||
kwargs[name].update(collection_to_kwargs(type_hints, set))
|
||||
elif contains_type(type_hints, int):
|
||||
kwargs[name]["type"] = int
|
||||
# Special case for large integers
|
||||
human_readable_ints = {
|
||||
"max_model_len",
|
||||
"max_num_batched_tokens",
|
||||
"kv_cache_memory_bytes",
|
||||
}
|
||||
if name in human_readable_ints:
|
||||
if name == "max_model_len":
|
||||
kwargs[name]["type"] = human_readable_int_or_auto
|
||||
kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
|
||||
elif name in ("max_num_batched_tokens", "kv_cache_memory_bytes"):
|
||||
kwargs[name]["type"] = human_readable_int
|
||||
kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
|
||||
else:
|
||||
kwargs[name]["type"] = int
|
||||
elif contains_type(type_hints, float):
|
||||
kwargs[name]["type"] = float
|
||||
elif contains_type(type_hints, dict) and (
|
||||
@@ -2042,23 +2040,17 @@ def _raise_unsupported_error(feature_name: str):
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
|
||||
def human_readable_int(value):
|
||||
def human_readable_int(value: str) -> int:
|
||||
"""Parse human-readable integers like '1k', '2M', etc.
|
||||
Including decimal values with decimal multipliers.
|
||||
Also accepts -1 or 'auto' as a special value for auto-detection.
|
||||
|
||||
Examples:
|
||||
- '1k' -> 1,000
|
||||
- '1K' -> 1,024
|
||||
- '25.6k' -> 25,600
|
||||
- '-1' or 'auto' -> -1 (special value for auto-detection)
|
||||
"""
|
||||
value = value.strip()
|
||||
|
||||
# Handle -1 or 'auto' as a special value for auto-detection
|
||||
if value == "-1" or value.lower() == "auto":
|
||||
return -1
|
||||
|
||||
match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value)
|
||||
if match:
|
||||
decimal_multiplier = {
|
||||
@@ -2092,3 +2084,22 @@ def human_readable_int(value):
|
||||
|
||||
# Regular plain number.
|
||||
return int(value)
|
||||
|
||||
|
||||
def human_readable_int_or_auto(value: str) -> int:
|
||||
"""Parse human-readable integers like '1k', '2M', etc.
|
||||
Including decimal values with decimal multipliers.
|
||||
Also accepts -1 or 'auto' as a special value for auto-detection.
|
||||
|
||||
Examples:
|
||||
- '1k' -> 1,000
|
||||
- '1K' -> 1,024
|
||||
- '25.6k' -> 25,600
|
||||
- '-1' or 'auto' -> -1 (special value for auto-detection)
|
||||
"""
|
||||
value = value.strip()
|
||||
|
||||
if value == "-1" or value.lower() == "auto":
|
||||
return -1
|
||||
|
||||
return human_readable_int(value)
|
||||
|
||||
Reference in New Issue
Block a user