[Perf][DSv4.1] Shard the Engram wkv projection across TP ranks (#58678)

Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
This commit is contained in:
Shuolei Wang
2026-09-26 01:15:57 +00:00
committed by GitHub
parent 8a2364605c
commit 4bb804cc5b
2 changed files with 7 additions and 3 deletions
+1 -1
View File
@@ -827,7 +827,7 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
engram_ops, "ReplicatedLinear", lambda *a, **k: torch.nn.Identity()
engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity()
)
stream = torch.cuda.Stream()
kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {}
+6 -2
View File
@@ -47,7 +47,7 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.utils import set_weight_attrs
from vllm.triton_utils import tl, triton
@@ -923,13 +923,17 @@ class Engram(nn.Module):
# suffix rule.
self.embed_tokens = self._create_embedding(layout, layer_hash_index)
n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads
self.wkv = ReplicatedLinear(
# Without sequence parallelism every TP rank holds every token, so
# shard the output columns instead of replicating the projection.
self.wkv = ColumnParallelLinear(
n_hash_cols * layout.head_dim,
self.dim * (self.hc_mult + 1),
bias=False,
gather_output=True,
quant_config=quant_config,
return_bias=False,
prefix=f"{prefix}.wkv",
disable_tp=use_sequence_parallel,
)
self.q_weight = nn.Parameter(
torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16),