[Model Loader] Apply EP weight filtering in InstantTensor (#61167)

Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Itay Etelis
2026-10-14 09:10:10 -07:00
committed by GitHub
co-authored by Robert Shaw Misha Goin
parent eaf25f675a
commit 13dbf96093
9 changed files with 84 additions and 6 deletions
@@ -318,6 +318,7 @@ class DefaultModelLoader(BaseModelLoader):
weights_iterator = instanttensor_weights_iterator(
hf_weights_files,
self.load_config.use_tqdm_on_load,
local_expert_ids=self.local_expert_ids,
)
else:
if extra_config.get("enable_multithread_load"):
@@ -1356,6 +1356,7 @@ def fastsafetensors_weights_iterator(
def instanttensor_weights_iterator(
hf_weights_files: list[str],
use_tqdm_on_load: bool,
local_expert_ids: set[int] | None = None,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files
using instanttensor library."""
@@ -1379,6 +1380,23 @@ def instanttensor_weights_iterator(
device = current_platform.current_device()
extra_kwargs: dict[str, Any] = {}
if local_expert_ids is not None:
checkpoint_names: list[str] = []
for path in hf_weights_files:
with safe_open(path, framework="pt") as metadata:
checkpoint_names.extend(metadata.keys()) # noqa: SIM118
# Every rank must make the same decision about collective loading.
empty_expert_ids: set[int] = set()
if any(should_skip_weight(name, empty_expert_ids) for name in checkpoint_names):
process_group = None
extra_kwargs["tensor_filter"] = [
name
for name in checkpoint_names
if not should_skip_weight(name, local_expert_ids)
]
extra_kwargs["tensor_name_match_mode"] = "exact"
# copy=True yields tensors that own their memory, staying valid after the
# context exits or InstantTensor reuses its buffer.
with instanttensor.safe_open(
@@ -1387,6 +1405,7 @@ def instanttensor_weights_iterator(
device=device,
process_group=process_group,
copy=True,
**extra_kwargs,
) as f:
# Track bytes so the bar reports load throughput (GB/s).
pbar = tqdm(