[Perf] Skip detokenization in offline beam search (#50333)

Signed-off-by: samuelkim7 <samuelmwkim@gmail.com>
2026-08-10 16:17:38 -07:00
parent 8bcc916a98
commit 8977ea8895
1 changed files with 2 additions and 0 deletions
@@ -119,6 +119,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
            logprobs=2 * beam_width,
            max_tokens=1,
            temperature=temperature,
+            detokenize=False,
            skip_clone=True,  # Internal beam search, safe to skip clone
        )
        instances: list[BeamSearchInstance] = []
@@ -443,6 +444,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
                logprobs=base_params.logprobs,
                max_tokens=1,
                temperature=base_params.temperature,
+                detokenize=False,
                allowed_token_ids=(
                    allowed_ids
                    if len(allowed_ids) <= _MAX_NUM_ALLOWED_TOKEN_IDS