[Model Loader] Apply EP weight filtering in InstantTensor (#61167)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Robert Shaw
Misha Goin
parent
eaf25f675a
commit
13dbf96093
@@ -318,6 +318,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
weights_iterator = instanttensor_weights_iterator(
|
||||
hf_weights_files,
|
||||
self.load_config.use_tqdm_on_load,
|
||||
local_expert_ids=self.local_expert_ids,
|
||||
)
|
||||
else:
|
||||
if extra_config.get("enable_multithread_load"):
|
||||
|
||||
@@ -1356,6 +1356,7 @@ def fastsafetensors_weights_iterator(
|
||||
def instanttensor_weights_iterator(
|
||||
hf_weights_files: list[str],
|
||||
use_tqdm_on_load: bool,
|
||||
local_expert_ids: set[int] | None = None,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files
|
||||
using instanttensor library."""
|
||||
@@ -1379,6 +1380,23 @@ def instanttensor_weights_iterator(
|
||||
|
||||
device = current_platform.current_device()
|
||||
|
||||
extra_kwargs: dict[str, Any] = {}
|
||||
if local_expert_ids is not None:
|
||||
checkpoint_names: list[str] = []
|
||||
for path in hf_weights_files:
|
||||
with safe_open(path, framework="pt") as metadata:
|
||||
checkpoint_names.extend(metadata.keys()) # noqa: SIM118
|
||||
# Every rank must make the same decision about collective loading.
|
||||
empty_expert_ids: set[int] = set()
|
||||
if any(should_skip_weight(name, empty_expert_ids) for name in checkpoint_names):
|
||||
process_group = None
|
||||
extra_kwargs["tensor_filter"] = [
|
||||
name
|
||||
for name in checkpoint_names
|
||||
if not should_skip_weight(name, local_expert_ids)
|
||||
]
|
||||
extra_kwargs["tensor_name_match_mode"] = "exact"
|
||||
|
||||
# copy=True yields tensors that own their memory, staying valid after the
|
||||
# context exits or InstantTensor reuses its buffer.
|
||||
with instanttensor.safe_open(
|
||||
@@ -1387,6 +1405,7 @@ def instanttensor_weights_iterator(
|
||||
device=device,
|
||||
process_group=process_group,
|
||||
copy=True,
|
||||
**extra_kwargs,
|
||||
) as f:
|
||||
# Track bytes so the bar reports load throughput (GB/s).
|
||||
pbar = tqdm(
|
||||
|
||||
Reference in New Issue
Block a user