[Speculative decoding 2/9] Multi-step worker for draft model (#2424)

This commit is contained in:
Cade Daniel
2024-01-21 16:31:47 -08:00
committed by GitHub
parent 71d63ed72e
commit 18bfcdd05c
11 changed files with 658 additions and 12 deletions
+4
View File
@@ -65,6 +65,10 @@ def get_ip() -> str:
return s.getsockname()[0]
def get_distributed_init_method(ip: str, port: int) -> str:
return f"tcp://{ip}:{port}"
def get_open_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("", 0))