[ModelRunnerV2] Support prompt embeds (#42963)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Signed-off-by: Canlin Guo <961750412@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Canlin Guo
2026-08-17 21:58:50 +00:00
committed by GitHub
co-authored by Nick Hill
parent e68fb75b25
commit 455edc022b
17 changed files with 551 additions and 54 deletions
+10
View File
@@ -68,6 +68,16 @@ NUM_BLOCKS = 10
DEVICE_TYPE = current_platform.device_type
@pytest.fixture(autouse=True)
def _restore_default_dtype():
"""Several tests here set the process-wide default dtype to float16 and
previously leaked it, corrupting later float-sensitive tests in the same
pytest process (torch.randn silently produced fp16)."""
old = torch.get_default_dtype()
yield
torch.set_default_dtype(old)
def initialize_kv_cache(runner: GPUModelRunner):
"""
Only perform necessary steps in GPUModelRunner.initialize_kv_cache()