diff --git a/tests/models/kimi_k3/test_kda_metadata.py b/tests/models/kimi_k3/test_kda_metadata.py index ca184467f0..ecf8d8e7c3 100644 --- a/tests/models/kimi_k3/test_kda_metadata.py +++ b/tests/models/kimi_k3/test_kda_metadata.py @@ -631,6 +631,68 @@ def test_kimi_k3_kda_cudagraph_capture_matches_shared_gdn(): _assert_matches_shared_gdn(reference, actual) +@pytest.mark.parametrize( + "builder_cls", + [ + GDNAttentionMetadataBuilder, + # The Kimi-K3 builder stages spec-decode state indices on the device. + pytest.param( + KimiK3KDAMetadataBuilder, + marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA" + ), + ), + ], +) +def test_cudagraph_capture_metadata_avoids_device_to_host_copy( + builder_cls: type[AttentionMetadataBuilder], monkeypatch: pytest.MonkeyPatch +): + """`build_for_cudagraph_capture` is not capture-only: a DP rank with nothing + scheduled re-stages its FULL-graph metadata through it on every dummy step, + so it must not synchronize the device. The host-side draft counts have to + come from `query_start_loc_cpu` and match the device-derived values.""" + num_speculative_tokens = 2 + batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3]) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + common_attn_metadata = create_common_attn_metadata( + batch, BLOCK_SIZE, device + ).replace(is_prefilling=torch.zeros(batch.batch_size, dtype=torch.bool)) + builder = _make_builder( + builder_cls, num_speculative_tokens, full_cuda_graph=True, device=device + ) + + device_to_host_copies: list[torch.Size] = [] + original_cpu = torch.Tensor.cpu + + def counting_cpu(self: torch.Tensor, *args, **kwargs): + device_to_host_copies.append(self.shape) + return original_cpu(self, *args, **kwargs) + + monkeypatch.setattr(torch.Tensor, "cpu", counting_cpu) + actual = builder.build_for_cudagraph_capture(common_attn_metadata) + monkeypatch.undo() + + assert not device_to_host_copies, device_to_host_copies + + num_accepted_tokens = torch.diff(common_attn_metadata.query_start_loc) + reference = builder.build( + 0, + common_attn_metadata, + num_accepted_tokens, + (num_accepted_tokens - 1).cpu(), + ) + assert actual.num_spec_decodes == batch.batch_size + assert actual.num_decodes == 0 + assert actual.num_prefills == 0 + for field in fields(type(actual)): + actual_value = getattr(actual, field.name) + reference_value = getattr(reference, field.name) + if isinstance(actual_value, torch.Tensor): + torch.testing.assert_close(actual_value, reference_value) + else: + assert actual_value == reference_value, field.name + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_recoverssm_spec_cudagraph_stages_one_checkpoint_per_request(): device = torch.device("cuda") diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index e4365a90c7..23daff5430 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -542,6 +542,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] ) num_accepted_tokens = torch.diff(m.query_start_loc) - num_decode_draft_tokens_cpu = (num_accepted_tokens - 1).cpu() + num_decode_draft_tokens_cpu = torch.diff(m.query_start_loc_cpu).sub_(1) + assert num_decode_draft_tokens_cpu.shape == num_accepted_tokens.shape return self.build(0, m, num_accepted_tokens, num_decode_draft_tokens_cpu) diff --git a/vllm/v1/attention/backends/short_conv_attn.py b/vllm/v1/attention/backends/short_conv_attn.py index 861f6cec48..5a0c6f7616 100644 --- a/vllm/v1/attention/backends/short_conv_attn.py +++ b/vllm/v1/attention/backends/short_conv_attn.py @@ -546,7 +546,9 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): if self.use_spec_decode: num_accepted_tokens = torch.diff(m.query_start_loc) - num_decode_draft_tokens_cpu = (num_accepted_tokens - 1).cpu() + + num_decode_draft_tokens_cpu = torch.diff(m.query_start_loc_cpu).sub_(1) + assert num_decode_draft_tokens_cpu.shape == num_accepted_tokens.shape return self.build( 0, m,