Revert "Fix Stale Encoder Cache After Weight Update" (#46125)
This commit is contained in:
@@ -898,12 +898,6 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
|||||||
def finish_weight_update(self) -> None:
|
def finish_weight_update(self) -> None:
|
||||||
"""Finish the current weight update."""
|
"""Finish the current weight update."""
|
||||||
self.llm_engine.collective_rpc("finish_weight_update")
|
self.llm_engine.collective_rpc("finish_weight_update")
|
||||||
# Invalidate cached state computed with the old weights so it isn't
|
|
||||||
# reused for subsequent requests:
|
|
||||||
# - prefix cache: KV blocks computed with the old weights
|
|
||||||
# - encoder cache: multimodal embeddings keyed only by mm_hash
|
|
||||||
self.llm_engine.reset_prefix_cache()
|
|
||||||
self.llm_engine.reset_encoder_cache()
|
|
||||||
|
|
||||||
def __repr__(self) -> str:
|
def __repr__(self) -> str:
|
||||||
"""Return a transformers-style hierarchical view of the model."""
|
"""Return a transformers-style hierarchical view of the model."""
|
||||||
|
|||||||
@@ -1109,9 +1109,3 @@ class AsyncLLM(EngineClient):
|
|||||||
async def finish_weight_update(self) -> None:
|
async def finish_weight_update(self) -> None:
|
||||||
"""Finish the current weight update."""
|
"""Finish the current weight update."""
|
||||||
await self.collective_rpc("finish_weight_update")
|
await self.collective_rpc("finish_weight_update")
|
||||||
# Invalidate cached state computed with the old weights so it isn't
|
|
||||||
# reused for subsequent requests:
|
|
||||||
# - prefix cache: KV blocks computed with the old weights
|
|
||||||
# - encoder cache: multimodal embeddings keyed only by mm_hash
|
|
||||||
await self.reset_prefix_cache()
|
|
||||||
await self.reset_encoder_cache()
|
|
||||||
|
|||||||
Reference in New Issue
Block a user