connection.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972
  1. #
  2. # A higher level module for using sockets (or Windows named pipes)
  3. #
  4. # multiprocessing/connection.py
  5. #
  6. # Copyright (c) 2006-2008, R Oudkerk
  7. # Licensed to PSF under a Contributor Agreement.
  8. #
  9. __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
  10. import io
  11. import os
  12. import sys
  13. import socket
  14. import struct
  15. import time
  16. import tempfile
  17. import itertools
  18. import _multiprocessing
  19. from . import util
  20. from . import AuthenticationError, BufferTooShort
  21. from .context import reduction
  22. _ForkingPickler = reduction.ForkingPickler
  23. try:
  24. import _winapi
  25. from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
  26. except ImportError:
  27. if sys.platform == 'win32':
  28. raise
  29. _winapi = None
  30. #
  31. #
  32. #
  33. BUFSIZE = 8192
  34. # A very generous timeout when it comes to local connections...
  35. CONNECTION_TIMEOUT = 20.
  36. _mmap_counter = itertools.count()
  37. default_family = 'AF_INET'
  38. families = ['AF_INET']
  39. if hasattr(socket, 'AF_UNIX'):
  40. default_family = 'AF_UNIX'
  41. families += ['AF_UNIX']
  42. if sys.platform == 'win32':
  43. default_family = 'AF_PIPE'
  44. families += ['AF_PIPE']
  45. def _init_timeout(timeout=CONNECTION_TIMEOUT):
  46. return time.monotonic() + timeout
  47. def _check_timeout(t):
  48. return time.monotonic() > t
  49. #
  50. #
  51. #
  52. def arbitrary_address(family):
  53. '''
  54. Return an arbitrary free address for the given family
  55. '''
  56. if family == 'AF_INET':
  57. return ('localhost', 0)
  58. elif family == 'AF_UNIX':
  59. return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
  60. elif family == 'AF_PIPE':
  61. return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
  62. (os.getpid(), next(_mmap_counter)), dir="")
  63. else:
  64. raise ValueError('unrecognized family')
  65. def _validate_family(family):
  66. '''
  67. Checks if the family is valid for the current environment.
  68. '''
  69. if sys.platform != 'win32' and family == 'AF_PIPE':
  70. raise ValueError('Family %s is not recognized.' % family)
  71. if sys.platform == 'win32' and family == 'AF_UNIX':
  72. # double check
  73. if not hasattr(socket, family):
  74. raise ValueError('Family %s is not recognized.' % family)
  75. def address_type(address):
  76. '''
  77. Return the types of the address
  78. This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
  79. '''
  80. if type(address) == tuple:
  81. return 'AF_INET'
  82. elif type(address) is str and address.startswith('\\\\'):
  83. return 'AF_PIPE'
  84. elif type(address) is str or util.is_abstract_socket_namespace(address):
  85. return 'AF_UNIX'
  86. else:
  87. raise ValueError('address type of %r unrecognized' % address)
  88. #
  89. # Connection classes
  90. #
  91. class _ConnectionBase:
  92. _handle = None
  93. def __init__(self, handle, readable=True, writable=True):
  94. handle = handle.__index__()
  95. if handle < 0:
  96. raise ValueError("invalid handle")
  97. if not readable and not writable:
  98. raise ValueError(
  99. "at least one of `readable` and `writable` must be True")
  100. self._handle = handle
  101. self._readable = readable
  102. self._writable = writable
  103. # XXX should we use util.Finalize instead of a __del__?
  104. def __del__(self):
  105. if self._handle is not None:
  106. self._close()
  107. def _check_closed(self):
  108. if self._handle is None:
  109. raise OSError("handle is closed")
  110. def _check_readable(self):
  111. if not self._readable:
  112. raise OSError("connection is write-only")
  113. def _check_writable(self):
  114. if not self._writable:
  115. raise OSError("connection is read-only")
  116. def _bad_message_length(self):
  117. if self._writable:
  118. self._readable = False
  119. else:
  120. self.close()
  121. raise OSError("bad message length")
  122. @property
  123. def closed(self):
  124. """True if the connection is closed"""
  125. return self._handle is None
  126. @property
  127. def readable(self):
  128. """True if the connection is readable"""
  129. return self._readable
  130. @property
  131. def writable(self):
  132. """True if the connection is writable"""
  133. return self._writable
  134. def fileno(self):
  135. """File descriptor or handle of the connection"""
  136. self._check_closed()
  137. return self._handle
  138. def close(self):
  139. """Close the connection"""
  140. if self._handle is not None:
  141. try:
  142. self._close()
  143. finally:
  144. self._handle = None
  145. def send_bytes(self, buf, offset=0, size=None):
  146. """Send the bytes data from a bytes-like object"""
  147. self._check_closed()
  148. self._check_writable()
  149. m = memoryview(buf)
  150. if m.itemsize > 1:
  151. m = m.cast('B')
  152. n = m.nbytes
  153. if offset < 0:
  154. raise ValueError("offset is negative")
  155. if n < offset:
  156. raise ValueError("buffer length < offset")
  157. if size is None:
  158. size = n - offset
  159. elif size < 0:
  160. raise ValueError("size is negative")
  161. elif offset + size > n:
  162. raise ValueError("buffer length < offset + size")
  163. self._send_bytes(m[offset:offset + size])
  164. def send(self, obj):
  165. """Send a (picklable) object"""
  166. self._check_closed()
  167. self._check_writable()
  168. self._send_bytes(_ForkingPickler.dumps(obj))
  169. def recv_bytes(self, maxlength=None):
  170. """
  171. Receive bytes data as a bytes object.
  172. """
  173. self._check_closed()
  174. self._check_readable()
  175. if maxlength is not None and maxlength < 0:
  176. raise ValueError("negative maxlength")
  177. buf = self._recv_bytes(maxlength)
  178. if buf is None:
  179. self._bad_message_length()
  180. return buf.getvalue()
  181. def recv_bytes_into(self, buf, offset=0):
  182. """
  183. Receive bytes data into a writeable bytes-like object.
  184. Return the number of bytes read.
  185. """
  186. self._check_closed()
  187. self._check_readable()
  188. with memoryview(buf) as m:
  189. # Get bytesize of arbitrary buffer
  190. itemsize = m.itemsize
  191. bytesize = itemsize * len(m)
  192. if offset < 0:
  193. raise ValueError("negative offset")
  194. elif offset > bytesize:
  195. raise ValueError("offset too large")
  196. result = self._recv_bytes()
  197. size = result.tell()
  198. if bytesize < offset + size:
  199. raise BufferTooShort(result.getvalue())
  200. # Message can fit in dest
  201. result.seek(0)
  202. result.readinto(m[offset // itemsize :
  203. (offset + size) // itemsize])
  204. return size
  205. def recv(self):
  206. """Receive a (picklable) object"""
  207. self._check_closed()
  208. self._check_readable()
  209. buf = self._recv_bytes()
  210. return _ForkingPickler.loads(buf.getbuffer())
  211. def poll(self, timeout=0.0):
  212. """Whether there is any input available to be read"""
  213. self._check_closed()
  214. self._check_readable()
  215. return self._poll(timeout)
  216. def __enter__(self):
  217. return self
  218. def __exit__(self, exc_type, exc_value, exc_tb):
  219. self.close()
  220. if _winapi:
  221. class PipeConnection(_ConnectionBase):
  222. """
  223. Connection class based on a Windows named pipe.
  224. Overlapped I/O is used, so the handles must have been created
  225. with FILE_FLAG_OVERLAPPED.
  226. """
  227. _got_empty_message = False
  228. def _close(self, _CloseHandle=_winapi.CloseHandle):
  229. _CloseHandle(self._handle)
  230. def _send_bytes(self, buf):
  231. ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
  232. try:
  233. if err == _winapi.ERROR_IO_PENDING:
  234. waitres = _winapi.WaitForMultipleObjects(
  235. [ov.event], False, INFINITE)
  236. assert waitres == WAIT_OBJECT_0
  237. except:
  238. ov.cancel()
  239. raise
  240. finally:
  241. nwritten, err = ov.GetOverlappedResult(True)
  242. assert err == 0
  243. assert nwritten == len(buf)
  244. def _recv_bytes(self, maxsize=None):
  245. if self._got_empty_message:
  246. self._got_empty_message = False
  247. return io.BytesIO()
  248. else:
  249. bsize = 128 if maxsize is None else min(maxsize, 128)
  250. try:
  251. ov, err = _winapi.ReadFile(self._handle, bsize,
  252. overlapped=True)
  253. try:
  254. if err == _winapi.ERROR_IO_PENDING:
  255. waitres = _winapi.WaitForMultipleObjects(
  256. [ov.event], False, INFINITE)
  257. assert waitres == WAIT_OBJECT_0
  258. except:
  259. ov.cancel()
  260. raise
  261. finally:
  262. nread, err = ov.GetOverlappedResult(True)
  263. if err == 0:
  264. f = io.BytesIO()
  265. f.write(ov.getbuffer())
  266. return f
  267. elif err == _winapi.ERROR_MORE_DATA:
  268. return self._get_more_data(ov, maxsize)
  269. except OSError as e:
  270. if e.winerror == _winapi.ERROR_BROKEN_PIPE:
  271. raise EOFError
  272. else:
  273. raise
  274. raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
  275. def _poll(self, timeout):
  276. if (self._got_empty_message or
  277. _winapi.PeekNamedPipe(self._handle)[0] != 0):
  278. return True
  279. return bool(wait([self], timeout))
  280. def _get_more_data(self, ov, maxsize):
  281. buf = ov.getbuffer()
  282. f = io.BytesIO()
  283. f.write(buf)
  284. left = _winapi.PeekNamedPipe(self._handle)[1]
  285. assert left > 0
  286. if maxsize is not None and len(buf) + left > maxsize:
  287. self._bad_message_length()
  288. ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
  289. rbytes, err = ov.GetOverlappedResult(True)
  290. assert err == 0
  291. assert rbytes == left
  292. f.write(ov.getbuffer())
  293. return f
  294. class Connection(_ConnectionBase):
  295. """
  296. Connection class based on an arbitrary file descriptor (Unix only), or
  297. a socket handle (Windows).
  298. """
  299. if _winapi:
  300. def _close(self, _close=_multiprocessing.closesocket):
  301. _close(self._handle)
  302. _write = _multiprocessing.send
  303. _read = _multiprocessing.recv
  304. else:
  305. def _close(self, _close=os.close):
  306. _close(self._handle)
  307. _write = os.write
  308. _read = os.read
  309. def _send(self, buf, write=_write):
  310. remaining = len(buf)
  311. while True:
  312. n = write(self._handle, buf)
  313. remaining -= n
  314. if remaining == 0:
  315. break
  316. buf = buf[n:]
  317. def _recv(self, size, read=_read):
  318. buf = io.BytesIO()
  319. handle = self._handle
  320. remaining = size
  321. while remaining > 0:
  322. chunk = read(handle, remaining)
  323. n = len(chunk)
  324. if n == 0:
  325. if remaining == size:
  326. raise EOFError
  327. else:
  328. raise OSError("got end of file during message")
  329. buf.write(chunk)
  330. remaining -= n
  331. return buf
  332. def _send_bytes(self, buf):
  333. n = len(buf)
  334. if n > 0x7fffffff:
  335. pre_header = struct.pack("!i", -1)
  336. header = struct.pack("!Q", n)
  337. self._send(pre_header)
  338. self._send(header)
  339. self._send(buf)
  340. else:
  341. # For wire compatibility with 3.7 and lower
  342. header = struct.pack("!i", n)
  343. if n > 16384:
  344. # The payload is large so Nagle's algorithm won't be triggered
  345. # and we'd better avoid the cost of concatenation.
  346. self._send(header)
  347. self._send(buf)
  348. else:
  349. # Issue #20540: concatenate before sending, to avoid delays due
  350. # to Nagle's algorithm on a TCP socket.
  351. # Also note we want to avoid sending a 0-length buffer separately,
  352. # to avoid "broken pipe" errors if the other end closed the pipe.
  353. self._send(header + buf)
  354. def _recv_bytes(self, maxsize=None):
  355. buf = self._recv(4)
  356. size, = struct.unpack("!i", buf.getvalue())
  357. if size == -1:
  358. buf = self._recv(8)
  359. size, = struct.unpack("!Q", buf.getvalue())
  360. if maxsize is not None and size > maxsize:
  361. return None
  362. return self._recv(size)
  363. def _poll(self, timeout):
  364. r = wait([self], timeout)
  365. return bool(r)
  366. #
  367. # Public functions
  368. #
  369. class Listener(object):
  370. '''
  371. Returns a listener object.
  372. This is a wrapper for a bound socket which is 'listening' for
  373. connections, or for a Windows named pipe.
  374. '''
  375. def __init__(self, address=None, family=None, backlog=1, authkey=None):
  376. family = family or (address and address_type(address)) \
  377. or default_family
  378. address = address or arbitrary_address(family)
  379. _validate_family(family)
  380. if family == 'AF_PIPE':
  381. self._listener = PipeListener(address, backlog)
  382. else:
  383. self._listener = SocketListener(address, family, backlog)
  384. if authkey is not None and not isinstance(authkey, bytes):
  385. raise TypeError('authkey should be a byte string')
  386. self._authkey = authkey
  387. def accept(self):
  388. '''
  389. Accept a connection on the bound socket or named pipe of `self`.
  390. Returns a `Connection` object.
  391. '''
  392. if self._listener is None:
  393. raise OSError('listener is closed')
  394. c = self._listener.accept()
  395. if self._authkey:
  396. deliver_challenge(c, self._authkey)
  397. answer_challenge(c, self._authkey)
  398. return c
  399. def close(self):
  400. '''
  401. Close the bound socket or named pipe of `self`.
  402. '''
  403. listener = self._listener
  404. if listener is not None:
  405. self._listener = None
  406. listener.close()
  407. @property
  408. def address(self):
  409. return self._listener._address
  410. @property
  411. def last_accepted(self):
  412. return self._listener._last_accepted
  413. def __enter__(self):
  414. return self
  415. def __exit__(self, exc_type, exc_value, exc_tb):
  416. self.close()
  417. def Client(address, family=None, authkey=None):
  418. '''
  419. Returns a connection to the address of a `Listener`
  420. '''
  421. family = family or address_type(address)
  422. _validate_family(family)
  423. if family == 'AF_PIPE':
  424. c = PipeClient(address)
  425. else:
  426. c = SocketClient(address)
  427. if authkey is not None and not isinstance(authkey, bytes):
  428. raise TypeError('authkey should be a byte string')
  429. if authkey is not None:
  430. answer_challenge(c, authkey)
  431. deliver_challenge(c, authkey)
  432. return c
  433. if sys.platform != 'win32':
  434. def Pipe(duplex=True):
  435. '''
  436. Returns pair of connection objects at either end of a pipe
  437. '''
  438. if duplex:
  439. s1, s2 = socket.socketpair()
  440. s1.setblocking(True)
  441. s2.setblocking(True)
  442. c1 = Connection(s1.detach())
  443. c2 = Connection(s2.detach())
  444. else:
  445. fd1, fd2 = os.pipe()
  446. c1 = Connection(fd1, writable=False)
  447. c2 = Connection(fd2, readable=False)
  448. return c1, c2
  449. else:
  450. def Pipe(duplex=True):
  451. '''
  452. Returns pair of connection objects at either end of a pipe
  453. '''
  454. address = arbitrary_address('AF_PIPE')
  455. if duplex:
  456. openmode = _winapi.PIPE_ACCESS_DUPLEX
  457. access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
  458. obsize, ibsize = BUFSIZE, BUFSIZE
  459. else:
  460. openmode = _winapi.PIPE_ACCESS_INBOUND
  461. access = _winapi.GENERIC_WRITE
  462. obsize, ibsize = 0, BUFSIZE
  463. h1 = _winapi.CreateNamedPipe(
  464. address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
  465. _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
  466. _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
  467. _winapi.PIPE_WAIT,
  468. 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
  469. # default security descriptor: the handle cannot be inherited
  470. _winapi.NULL
  471. )
  472. h2 = _winapi.CreateFile(
  473. address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
  474. _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
  475. )
  476. _winapi.SetNamedPipeHandleState(
  477. h2, _winapi.PIPE_READMODE_MESSAGE, None, None
  478. )
  479. overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
  480. _, err = overlapped.GetOverlappedResult(True)
  481. assert err == 0
  482. c1 = PipeConnection(h1, writable=duplex)
  483. c2 = PipeConnection(h2, readable=duplex)
  484. return c1, c2
  485. #
  486. # Definitions for connections based on sockets
  487. #
  488. class SocketListener(object):
  489. '''
  490. Representation of a socket which is bound to an address and listening
  491. '''
  492. def __init__(self, address, family, backlog=1):
  493. self._socket = socket.socket(getattr(socket, family))
  494. try:
  495. # SO_REUSEADDR has different semantics on Windows (issue #2550).
  496. if os.name == 'posix':
  497. self._socket.setsockopt(socket.SOL_SOCKET,
  498. socket.SO_REUSEADDR, 1)
  499. self._socket.setblocking(True)
  500. self._socket.bind(address)
  501. self._socket.listen(backlog)
  502. self._address = self._socket.getsockname()
  503. except OSError:
  504. self._socket.close()
  505. raise
  506. self._family = family
  507. self._last_accepted = None
  508. if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address):
  509. # Linux abstract socket namespaces do not need to be explicitly unlinked
  510. self._unlink = util.Finalize(
  511. self, os.unlink, args=(address,), exitpriority=0
  512. )
  513. else:
  514. self._unlink = None
  515. def accept(self):
  516. s, self._last_accepted = self._socket.accept()
  517. s.setblocking(True)
  518. return Connection(s.detach())
  519. def close(self):
  520. try:
  521. self._socket.close()
  522. finally:
  523. unlink = self._unlink
  524. if unlink is not None:
  525. self._unlink = None
  526. unlink()
  527. def SocketClient(address):
  528. '''
  529. Return a connection object connected to the socket given by `address`
  530. '''
  531. family = address_type(address)
  532. with socket.socket( getattr(socket, family) ) as s:
  533. s.setblocking(True)
  534. s.connect(address)
  535. return Connection(s.detach())
  536. #
  537. # Definitions for connections based on named pipes
  538. #
  539. if sys.platform == 'win32':
  540. class PipeListener(object):
  541. '''
  542. Representation of a named pipe
  543. '''
  544. def __init__(self, address, backlog=None):
  545. self._address = address
  546. self._handle_queue = [self._new_handle(first=True)]
  547. self._last_accepted = None
  548. util.sub_debug('listener created with address=%r', self._address)
  549. self.close = util.Finalize(
  550. self, PipeListener._finalize_pipe_listener,
  551. args=(self._handle_queue, self._address), exitpriority=0
  552. )
  553. def _new_handle(self, first=False):
  554. flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
  555. if first:
  556. flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
  557. return _winapi.CreateNamedPipe(
  558. self._address, flags,
  559. _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
  560. _winapi.PIPE_WAIT,
  561. _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
  562. _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
  563. )
  564. def accept(self):
  565. self._handle_queue.append(self._new_handle())
  566. handle = self._handle_queue.pop(0)
  567. try:
  568. ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
  569. except OSError as e:
  570. if e.winerror != _winapi.ERROR_NO_DATA:
  571. raise
  572. # ERROR_NO_DATA can occur if a client has already connected,
  573. # written data and then disconnected -- see Issue 14725.
  574. else:
  575. try:
  576. res = _winapi.WaitForMultipleObjects(
  577. [ov.event], False, INFINITE)
  578. except:
  579. ov.cancel()
  580. _winapi.CloseHandle(handle)
  581. raise
  582. finally:
  583. _, err = ov.GetOverlappedResult(True)
  584. assert err == 0
  585. return PipeConnection(handle)
  586. @staticmethod
  587. def _finalize_pipe_listener(queue, address):
  588. util.sub_debug('closing listener with address=%r', address)
  589. for handle in queue:
  590. _winapi.CloseHandle(handle)
  591. def PipeClient(address):
  592. '''
  593. Return a connection object connected to the pipe given by `address`
  594. '''
  595. t = _init_timeout()
  596. while 1:
  597. try:
  598. _winapi.WaitNamedPipe(address, 1000)
  599. h = _winapi.CreateFile(
  600. address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
  601. 0, _winapi.NULL, _winapi.OPEN_EXISTING,
  602. _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
  603. )
  604. except OSError as e:
  605. if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
  606. _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
  607. raise
  608. else:
  609. break
  610. else:
  611. raise
  612. _winapi.SetNamedPipeHandleState(
  613. h, _winapi.PIPE_READMODE_MESSAGE, None, None
  614. )
  615. return PipeConnection(h)
  616. #
  617. # Authentication stuff
  618. #
  619. MESSAGE_LENGTH = 20
  620. CHALLENGE = b'#CHALLENGE#'
  621. WELCOME = b'#WELCOME#'
  622. FAILURE = b'#FAILURE#'
  623. def deliver_challenge(connection, authkey):
  624. import hmac
  625. if not isinstance(authkey, bytes):
  626. raise ValueError(
  627. "Authkey must be bytes, not {0!s}".format(type(authkey)))
  628. message = os.urandom(MESSAGE_LENGTH)
  629. connection.send_bytes(CHALLENGE + message)
  630. digest = hmac.new(authkey, message, 'md5').digest()
  631. response = connection.recv_bytes(256) # reject large message
  632. if response == digest:
  633. connection.send_bytes(WELCOME)
  634. else:
  635. connection.send_bytes(FAILURE)
  636. raise AuthenticationError('digest received was wrong')
  637. def answer_challenge(connection, authkey):
  638. import hmac
  639. if not isinstance(authkey, bytes):
  640. raise ValueError(
  641. "Authkey must be bytes, not {0!s}".format(type(authkey)))
  642. message = connection.recv_bytes(256) # reject large message
  643. assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
  644. message = message[len(CHALLENGE):]
  645. digest = hmac.new(authkey, message, 'md5').digest()
  646. connection.send_bytes(digest)
  647. response = connection.recv_bytes(256) # reject large message
  648. if response != WELCOME:
  649. raise AuthenticationError('digest sent was rejected')
  650. #
  651. # Support for using xmlrpclib for serialization
  652. #
  653. class ConnectionWrapper(object):
  654. def __init__(self, conn, dumps, loads):
  655. self._conn = conn
  656. self._dumps = dumps
  657. self._loads = loads
  658. for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
  659. obj = getattr(conn, attr)
  660. setattr(self, attr, obj)
  661. def send(self, obj):
  662. s = self._dumps(obj)
  663. self._conn.send_bytes(s)
  664. def recv(self):
  665. s = self._conn.recv_bytes()
  666. return self._loads(s)
  667. def _xml_dumps(obj):
  668. return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
  669. def _xml_loads(s):
  670. (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
  671. return obj
  672. class XmlListener(Listener):
  673. def accept(self):
  674. global xmlrpclib
  675. import xmlrpc.client as xmlrpclib
  676. obj = Listener.accept(self)
  677. return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
  678. def XmlClient(*args, **kwds):
  679. global xmlrpclib
  680. import xmlrpc.client as xmlrpclib
  681. return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
  682. #
  683. # Wait
  684. #
  685. if sys.platform == 'win32':
  686. def _exhaustive_wait(handles, timeout):
  687. # Return ALL handles which are currently signalled. (Only
  688. # returning the first signalled might create starvation issues.)
  689. L = list(handles)
  690. ready = []
  691. while L:
  692. res = _winapi.WaitForMultipleObjects(L, False, timeout)
  693. if res == WAIT_TIMEOUT:
  694. break
  695. elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
  696. res -= WAIT_OBJECT_0
  697. elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
  698. res -= WAIT_ABANDONED_0
  699. else:
  700. raise RuntimeError('Should not get here')
  701. ready.append(L[res])
  702. L = L[res+1:]
  703. timeout = 0
  704. return ready
  705. _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
  706. def wait(object_list, timeout=None):
  707. '''
  708. Wait till an object in object_list is ready/readable.
  709. Returns list of those objects in object_list which are ready/readable.
  710. '''
  711. if timeout is None:
  712. timeout = INFINITE
  713. elif timeout < 0:
  714. timeout = 0
  715. else:
  716. timeout = int(timeout * 1000 + 0.5)
  717. object_list = list(object_list)
  718. waithandle_to_obj = {}
  719. ov_list = []
  720. ready_objects = set()
  721. ready_handles = set()
  722. try:
  723. for o in object_list:
  724. try:
  725. fileno = getattr(o, 'fileno')
  726. except AttributeError:
  727. waithandle_to_obj[o.__index__()] = o
  728. else:
  729. # start an overlapped read of length zero
  730. try:
  731. ov, err = _winapi.ReadFile(fileno(), 0, True)
  732. except OSError as e:
  733. ov, err = None, e.winerror
  734. if err not in _ready_errors:
  735. raise
  736. if err == _winapi.ERROR_IO_PENDING:
  737. ov_list.append(ov)
  738. waithandle_to_obj[ov.event] = o
  739. else:
  740. # If o.fileno() is an overlapped pipe handle and
  741. # err == 0 then there is a zero length message
  742. # in the pipe, but it HAS NOT been consumed...
  743. if ov and sys.getwindowsversion()[:2] >= (6, 2):
  744. # ... except on Windows 8 and later, where
  745. # the message HAS been consumed.
  746. try:
  747. _, err = ov.GetOverlappedResult(False)
  748. except OSError as e:
  749. err = e.winerror
  750. if not err and hasattr(o, '_got_empty_message'):
  751. o._got_empty_message = True
  752. ready_objects.add(o)
  753. timeout = 0
  754. ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
  755. finally:
  756. # request that overlapped reads stop
  757. for ov in ov_list:
  758. ov.cancel()
  759. # wait for all overlapped reads to stop
  760. for ov in ov_list:
  761. try:
  762. _, err = ov.GetOverlappedResult(True)
  763. except OSError as e:
  764. err = e.winerror
  765. if err not in _ready_errors:
  766. raise
  767. if err != _winapi.ERROR_OPERATION_ABORTED:
  768. o = waithandle_to_obj[ov.event]
  769. ready_objects.add(o)
  770. if err == 0:
  771. # If o.fileno() is an overlapped pipe handle then
  772. # a zero length message HAS been consumed.
  773. if hasattr(o, '_got_empty_message'):
  774. o._got_empty_message = True
  775. ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
  776. return [o for o in object_list if o in ready_objects]
  777. else:
  778. import selectors
  779. # poll/select have the advantage of not requiring any extra file
  780. # descriptor, contrarily to epoll/kqueue (also, they require a single
  781. # syscall).
  782. if hasattr(selectors, 'PollSelector'):
  783. _WaitSelector = selectors.PollSelector
  784. else:
  785. _WaitSelector = selectors.SelectSelector
  786. def wait(object_list, timeout=None):
  787. '''
  788. Wait till an object in object_list is ready/readable.
  789. Returns list of those objects in object_list which are ready/readable.
  790. '''
  791. with _WaitSelector() as selector:
  792. for obj in object_list:
  793. selector.register(obj, selectors.EVENT_READ)
  794. if timeout is not None:
  795. deadline = time.monotonic() + timeout
  796. while True:
  797. ready = selector.select(timeout)
  798. if ready:
  799. return [key.fileobj for (key, events) in ready]
  800. else:
  801. if timeout is not None:
  802. timeout = deadline - time.monotonic()
  803. if timeout < 0:
  804. return ready
  805. #
  806. # Make connection and socket objects shareable if possible
  807. #
  808. if sys.platform == 'win32':
  809. def reduce_connection(conn):
  810. handle = conn.fileno()
  811. with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
  812. from . import resource_sharer
  813. ds = resource_sharer.DupSocket(s)
  814. return rebuild_connection, (ds, conn.readable, conn.writable)
  815. def rebuild_connection(ds, readable, writable):
  816. sock = ds.detach()
  817. return Connection(sock.detach(), readable, writable)
  818. reduction.register(Connection, reduce_connection)
  819. def reduce_pipe_connection(conn):
  820. access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
  821. (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
  822. dh = reduction.DupHandle(conn.fileno(), access)
  823. return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
  824. def rebuild_pipe_connection(dh, readable, writable):
  825. handle = dh.detach()
  826. return PipeConnection(handle, readable, writable)
  827. reduction.register(PipeConnection, reduce_pipe_connection)
  828. else:
  829. def reduce_connection(conn):
  830. df = reduction.DupFd(conn.fileno())
  831. return rebuild_connection, (df, conn.readable, conn.writable)
  832. def rebuild_connection(df, readable, writable):
  833. fd = df.detach()
  834. return Connection(fd, readable, writable)
  835. reduction.register(Connection, reduce_connection)