[ROCm][V1][MLA] Clone prefill backend state per metadata builder (#46993)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -28,6 +28,28 @@ class CustomMLAPrefillBackend(MLAPrefillBackend):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def test_prefill_backend_clone_has_isolated_metadata():
|
||||
backend = CustomMLAPrefillBackend(
|
||||
num_heads=4,
|
||||
scale=0.5,
|
||||
kv_lora_rank=8,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=8,
|
||||
v_head_dim=32,
|
||||
vllm_config=object(),
|
||||
)
|
||||
|
||||
clone = backend.clone()
|
||||
|
||||
assert isinstance(clone, CustomMLAPrefillBackend)
|
||||
assert clone is not backend
|
||||
assert clone.num_heads == backend.num_heads
|
||||
assert clone.scale == backend.scale
|
||||
backend._prefill_metadata = object()
|
||||
clone._prefill_metadata = object()
|
||||
assert clone._prefill_metadata is not backend._prefill_metadata
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_overrides():
|
||||
"""Clear any overrides after each test."""
|
||||
|
||||
@@ -1563,9 +1563,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Metadata builders are created per ubatch when DBO is enabled. MLA
|
||||
# prefill backends keep the prepared metadata on the backend object, so
|
||||
# each builder needs its own backend instance to avoid cross-ubatch races.
|
||||
self._prefill_backend = self.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend
|
||||
].prefill_backend.clone()
|
||||
|
||||
supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY
|
||||
self._init_reorder_batch_threshold(
|
||||
|
||||
@@ -116,6 +116,17 @@ class MLAPrefillBackend(ABC):
|
||||
self.v_head_dim = v_head_dim
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
def clone(self) -> "MLAPrefillBackend":
|
||||
return self.__class__(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.scale,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
vllm_config=self.vllm_config,
|
||||
)
|
||||
|
||||
def prepare_metadata( # noqa: B027
|
||||
self,
|
||||
prefill_metadata: "MLACommonPrefillMetadata",
|
||||
|
||||
Reference in New Issue
Block a user