Hidden states extraction improvements (#43805)

Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Fynn Schmitt-Ulms
2026-06-11 09:44:45 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab3a1fd2e6
commit ebc6ef971a
7 changed files with 454 additions and 233 deletions
@@ -92,7 +92,6 @@ def run_baseline(
llm = LLM(
model=model,
enable_prefix_caching=False,
enable_chunked_prefill=False,
**extra_args,
)
sampling_params = SamplingParams(max_tokens=1)
@@ -194,7 +193,6 @@ async def _run_extraction_async(
engine_args = AsyncEngineArgs(
model=model,
enable_prefix_caching=False,
enable_chunked_prefill=False,
max_num_batched_tokens=40960,
max_model_len=40960,
speculative_config={