[Frontend] Add reusable TP1 initialized-engine snapshots (#51360)
Signed-off-by: Nils Matteson <nilsmatteson@icloud.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: elehayym <52448798+Yuzu23@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Taneem Ibrahim
elehayym
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8bf051cd51
commit
13058faa49
@@ -19,6 +19,7 @@ import torch
|
||||
|
||||
from vllm.model_executor.models.mimo_v2 import _shard_fp8_qkv_proj
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -39,6 +40,13 @@ GEOMETRIES = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_threaded_cpu_ops():
|
||||
# Do not prime the parent's thread pool before later engine forks.
|
||||
with set_default_torch_num_threads(1):
|
||||
yield
|
||||
|
||||
|
||||
def _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp):
|
||||
return (
|
||||
num_heads // ckpt_tp * head_dim
|
||||
|
||||
Reference in New Issue
Block a user