[Perf] Optimize hidden state extraction logic (#37374)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
f743254143
commit
4e2eba28be
@@ -120,6 +120,7 @@ def test_extract_hidden_states_with_predictable_dummy_model(
|
||||
},
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=False,
|
||||
trust_remote_code=True,
|
||||
load_format="dummy", # Don't try to load real weights
|
||||
)
|
||||
@@ -184,6 +185,7 @@ def test_extract_hidden_states_qwen35_hybrid_smoke(tmp_path):
|
||||
},
|
||||
max_model_len=256,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=False,
|
||||
gpu_memory_utilization=0.4,
|
||||
load_format="dummy",
|
||||
)
|
||||
|
||||
@@ -69,6 +69,7 @@ def _create_proposer(
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
enable_chunked_prefill=False,
|
||||
),
|
||||
attention_config=AttentionConfig(),
|
||||
)
|
||||
@@ -119,6 +120,7 @@ def test_proposer_initialization_missing_layer_ids():
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
enable_chunked_prefill=False,
|
||||
),
|
||||
attention_config=AttentionConfig(),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user