[v0][structured output] Support reasoning output (#12955)
Signed-off-by: Ce Gao <cegao@tensorchord.ai>
This commit is contained in:
@@ -213,6 +213,8 @@ class EngineArgs:
|
||||
calculate_kv_scales: Optional[bool] = None
|
||||
|
||||
additional_config: Optional[Dict[str, Any]] = None
|
||||
enable_reasoning: Optional[bool] = None
|
||||
reasoning_parser: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.tokenizer:
|
||||
@@ -1059,6 +1061,25 @@ class EngineArgs:
|
||||
"Different platforms may support different configs. Make sure the "
|
||||
"configs are valid for the platform you are using. The input format"
|
||||
" is like '{\"config_key\":\"config_value\"}'")
|
||||
|
||||
parser.add_argument(
|
||||
"--enable-reasoning",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Whether to enable reasoning_content for the model. "
|
||||
"If enabled, the model will be able to generate reasoning content."
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--reasoning-parser",
|
||||
type=str,
|
||||
choices=["deepseek_r1"],
|
||||
default=None,
|
||||
help=
|
||||
"Select the reasoning parser depending on the model that you're "
|
||||
"using. This is used to parse the reasoning content into OpenAI "
|
||||
"API format. Required for ``--enable-reasoning``.")
|
||||
|
||||
return parser
|
||||
|
||||
@classmethod
|
||||
@@ -1332,7 +1353,10 @@ class EngineArgs:
|
||||
if self.enable_prompt_adapter else None
|
||||
|
||||
decoding_config = DecodingConfig(
|
||||
guided_decoding_backend=self.guided_decoding_backend)
|
||||
guided_decoding_backend=self.guided_decoding_backend,
|
||||
reasoning_backend=self.reasoning_parser
|
||||
if self.enable_reasoning else None,
|
||||
)
|
||||
|
||||
show_hidden_metrics = False
|
||||
if self.show_hidden_metrics_for_version is not None:
|
||||
|
||||
@@ -509,6 +509,7 @@ class _AsyncLLMEngine(LLMEngine):
|
||||
tokenizer=await self.get_tokenizer_async(lora_request),
|
||||
default_guided_backend=self.decoding_config.
|
||||
guided_decoding_backend,
|
||||
reasoning_backend=self.decoding_config.reasoning_backend,
|
||||
model_config=self.model_config)
|
||||
|
||||
self._add_processed_request(
|
||||
@@ -530,7 +531,7 @@ class _AsyncLLMEngine(LLMEngine):
|
||||
|
||||
async def build_guided_decoding_logits_processor_async(
|
||||
sampling_params: SamplingParams, tokenizer: AnyTokenizer,
|
||||
default_guided_backend: str,
|
||||
default_guided_backend: str, reasoning_backend: Optional[str],
|
||||
model_config: ModelConfig) -> SamplingParams:
|
||||
"""Constructs logits processors based on the guided_decoding,
|
||||
logits_bias, and allowed_token_ids fields in sampling_params. Deletes
|
||||
@@ -545,14 +546,18 @@ async def build_guided_decoding_logits_processor_async(
|
||||
sampling_params = copy.copy(sampling_params)
|
||||
guided_decoding = sampling_params.guided_decoding
|
||||
|
||||
logger.debug("Building guided decoding logits processor. "
|
||||
"Params: %s", guided_decoding)
|
||||
logger.info(
|
||||
"Building guided decoding logits processor. "
|
||||
"guided_decoding: %s%s", guided_decoding,
|
||||
f", reasoning_backend: {reasoning_backend}"
|
||||
if reasoning_backend is not None else "")
|
||||
|
||||
guided_decoding.backend = guided_decoding.backend or default_guided_backend
|
||||
|
||||
processor = await get_guided_decoding_logits_processor(
|
||||
guided_params=guided_decoding,
|
||||
tokenizer=tokenizer,
|
||||
reasoning_backend=reasoning_backend,
|
||||
model_config=model_config)
|
||||
|
||||
if processor:
|
||||
|
||||
@@ -2048,10 +2048,15 @@ class LLMEngine:
|
||||
guided_decoding.backend = guided_decoding.backend or \
|
||||
self.decoding_config.guided_decoding_backend
|
||||
|
||||
logger.debug("Reasoning backend: %s",
|
||||
self.decoding_config.reasoning_backend)
|
||||
|
||||
processor = get_local_guided_decoding_logits_processor(
|
||||
guided_params=guided_decoding,
|
||||
tokenizer=tokenizer,
|
||||
model_config=self.model_config)
|
||||
model_config=self.model_config,
|
||||
reasoning_backend=self.decoding_config.reasoning_backend,
|
||||
)
|
||||
if processor:
|
||||
logits_processors.append(processor)
|
||||
|
||||
|
||||
@@ -611,7 +611,8 @@ class MQLLMEngineClient(EngineClient):
|
||||
default_guided_backend=(self.decoding_config.guided_decoding_backend
|
||||
if self.decoding_config
|
||||
else DecodingConfig.guided_decoding_backend),
|
||||
model_config=self.model_config
|
||||
model_config=self.model_config,
|
||||
reasoning_backend=self.decoding_config.reasoning_backend,
|
||||
)
|
||||
|
||||
# 1) Create output queue for this requests.
|
||||
|
||||
Reference in New Issue
Block a user