[Frontend] Customizable RoPE theta (#5197)
This commit is contained in:
@@ -53,6 +53,7 @@ class EngineArgs:
|
||||
revision: Optional[str] = None
|
||||
code_revision: Optional[str] = None
|
||||
rope_scaling: Optional[dict] = None
|
||||
rope_theta: Optional[float] = None
|
||||
tokenizer_revision: Optional[str] = None
|
||||
quantization: Optional[str] = None
|
||||
enforce_eager: bool = False
|
||||
@@ -400,6 +401,12 @@ class EngineArgs:
|
||||
type=json.loads,
|
||||
help='RoPE scaling configuration in JSON format. '
|
||||
'For example, {"type":"dynamic","factor":2.0}')
|
||||
parser.add_argument('--rope-theta',
|
||||
default=None,
|
||||
type=float,
|
||||
help='RoPE theta. Use with `rope_scaling`. In '
|
||||
'some cases, changing the RoPE theta improves the '
|
||||
'performance of the scaled model.')
|
||||
parser.add_argument('--enforce-eager',
|
||||
action='store_true',
|
||||
help='Always use eager-mode PyTorch. If False, '
|
||||
@@ -630,6 +637,7 @@ class EngineArgs:
|
||||
revision=self.revision,
|
||||
code_revision=self.code_revision,
|
||||
rope_scaling=self.rope_scaling,
|
||||
rope_theta=self.rope_theta,
|
||||
tokenizer_revision=self.tokenizer_revision,
|
||||
max_model_len=self.max_model_len,
|
||||
quantization=self.quantization,
|
||||
|
||||
@@ -162,7 +162,7 @@ class LLMEngine:
|
||||
"Initializing an LLM engine (v%s) with config: "
|
||||
"model=%r, speculative_config=%r, tokenizer=%r, "
|
||||
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
|
||||
"rope_scaling=%r, tokenizer_revision=%s, "
|
||||
"rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
|
||||
"trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
|
||||
"download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
|
||||
"disable_custom_all_reduce=%s, quantization=%s, "
|
||||
@@ -177,6 +177,7 @@ class LLMEngine:
|
||||
model_config.tokenizer_mode,
|
||||
model_config.revision,
|
||||
model_config.rope_scaling,
|
||||
model_config.rope_theta,
|
||||
model_config.tokenizer_revision,
|
||||
model_config.trust_remote_code,
|
||||
model_config.dtype,
|
||||
|
||||
Reference in New Issue
Block a user