diff --git a/tests/kernels/test_engram.py b/tests/kernels/test_engram.py index 4003c0516a..9f62b0d438 100644 --- a/tests/kernels/test_engram.py +++ b/tests/kernels/test_engram.py @@ -827,7 +827,7 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload): monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1) monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0) monkeypatch.setattr( - engram_ops, "ReplicatedLinear", lambda *a, **k: torch.nn.Identity() + engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity() ) stream = torch.cuda.Stream() kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {} diff --git a/vllm/models/deepseek_v41/common/engram.py b/vllm/models/deepseek_v41/common/engram.py index c415bbaa6e..8cd93547e6 100644 --- a/vllm/models/deepseek_v41/common/engram.py +++ b/vllm/models/deepseek_v41/common/engram.py @@ -47,7 +47,7 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, ) from vllm.logger import init_logger -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.utils import set_weight_attrs from vllm.triton_utils import tl, triton @@ -923,13 +923,17 @@ class Engram(nn.Module): # suffix rule. self.embed_tokens = self._create_embedding(layout, layer_hash_index) n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads - self.wkv = ReplicatedLinear( + # Without sequence parallelism every TP rank holds every token, so + # shard the output columns instead of replicating the projection. + self.wkv = ColumnParallelLinear( n_hash_cols * layout.head_dim, self.dim * (self.hc_mult + 1), bias=False, + gather_output=True, quant_config=quant_config, return_bias=False, prefix=f"{prefix}.wkv", + disable_tp=use_sequence_parallel, ) self.q_weight = nn.Parameter( torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16),