connection.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850
  1. # -*- coding: utf-8 -*-
  2. """
  3. hyper/http20/connection
  4. ~~~~~~~~~~~~~~~~~~~~~~~
  5. Objects that build hyper's connection-level HTTP/2 abstraction.
  6. """
  7. import h2.connection
  8. import h2.events
  9. import h2.settings
  10. from ..compat import ssl
  11. from ..tls import wrap_socket, H2_NPN_PROTOCOLS, H2C_PROTOCOL
  12. from ..common.exceptions import ConnectionResetError
  13. from ..common.bufsocket import BufferedSocket
  14. from ..common.headers import HTTPHeaderMap
  15. from ..common.util import to_host_port_tuple, to_native_string, to_bytestring
  16. from ..compat import unicode, bytes
  17. from .stream import Stream
  18. from .response import HTTP20Response, HTTP20Push
  19. from .window import FlowControlManager
  20. from .exceptions import ConnectionError, StreamResetError
  21. from . import errors
  22. import errno
  23. import logging
  24. import socket
  25. import time
  26. import threading
  27. log = logging.getLogger(__name__)
  28. DEFAULT_WINDOW_SIZE = 65535
  29. TRANSIENT_SSL_ERRORS = (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE)
  30. class _LockedObject(object):
  31. """
  32. A wrapper class that hides a specific object behind a lock.
  33. The goal here is to provide a simple way to protect access to an object
  34. that cannot safely be simultaneously accessed from multiple threads. The
  35. intended use of this class is simple: take hold of it with a context
  36. manager, which returns the protected object.
  37. """
  38. def __init__(self, obj):
  39. self.lock = threading.RLock()
  40. self._obj = obj
  41. def __enter__(self):
  42. self.lock.acquire()
  43. return self._obj
  44. def __exit__(self, _exc_type, _exc_val, _exc_tb):
  45. self.lock.release()
  46. class HTTP20Connection(object):
  47. """
  48. An object representing a single HTTP/2 connection to a server.
  49. This object behaves similarly to the Python standard library's
  50. ``HTTPConnection`` object, with a few critical differences.
  51. Most of the standard library's arguments to the constructor are irrelevant
  52. for HTTP/2 or not supported by hyper.
  53. :param host: The host to connect to. This may be an IP address or a
  54. hostname, and optionally may include a port: for example,
  55. ``'http2bin.org'``, ``'http2bin.org:443'`` or ``'127.0.0.1'``.
  56. :param port: (optional) The port to connect to. If not provided and one
  57. also isn't provided in the ``host`` parameter, defaults to 443.
  58. :param secure: (optional) Whether the request should use TLS. Defaults to
  59. ``False`` for most requests, but to ``True`` for any request issued to
  60. port 443.
  61. :param window_manager: (optional) The class to use to manage flow control
  62. windows. This needs to be a subclass of the
  63. :class:`BaseFlowControlManager
  64. <hyper.http20.window.BaseFlowControlManager>`. If not provided,
  65. :class:`FlowControlManager <hyper.http20.window.FlowControlManager>`
  66. will be used.
  67. :param enable_push: (optional) Whether the server is allowed to push
  68. resources to the client (see
  69. :meth:`get_pushes() <hyper.HTTP20Connection.get_pushes>`).
  70. :param ssl_context: (optional) A class with custom certificate settings.
  71. If not provided then hyper's default ``SSLContext`` is used instead.
  72. :param proxy_host: (optional) The proxy to connect to. This can be an IP
  73. address or a host name and may include a port.
  74. :param proxy_port: (optional) The proxy port to connect to. If not provided
  75. and one also isn't provided in the ``proxy`` parameter, defaults to
  76. 8080.
  77. """
  78. def __init__(self, host, port=None, secure=None, window_manager=None,
  79. enable_push=False, ssl_context=None, proxy_host=None,
  80. proxy_port=None, force_proto=None, **kwargs):
  81. """
  82. Creates an HTTP/2 connection to a specific server.
  83. """
  84. if port is None:
  85. self.host, self.port = to_host_port_tuple(host, default_port=443)
  86. else:
  87. self.host, self.port = host, port
  88. if secure is not None:
  89. self.secure = secure
  90. elif self.port == 443:
  91. self.secure = True
  92. else:
  93. self.secure = False
  94. self._enable_push = enable_push
  95. self.ssl_context = ssl_context
  96. # Setup proxy details if applicable.
  97. if proxy_host:
  98. if proxy_port is None:
  99. self.proxy_host, self.proxy_port = to_host_port_tuple(
  100. proxy_host, default_port=8080
  101. )
  102. else:
  103. self.proxy_host, self.proxy_port = proxy_host, proxy_port
  104. else:
  105. self.proxy_host = None
  106. self.proxy_port = None
  107. #: The size of the in-memory buffer used to store data from the
  108. #: network. This is used as a performance optimisation. Increase buffer
  109. #: size to improve performance: decrease it to conserve memory.
  110. #: Defaults to 64kB.
  111. self.network_buffer_size = 65536
  112. self.force_proto = force_proto
  113. # Concurrency
  114. #
  115. # Use one lock (_lock) to synchronize any interaction with global
  116. # connection state, e.g. stream creation/deletion.
  117. #
  118. # It's ok to use the same in lock all these cases as they occur at
  119. # different/linked points in the connection's lifecycle.
  120. #
  121. # Use another 2 locks (_write_lock, _read_lock) to synchronize
  122. # - _send_cb
  123. # - _recv_cb
  124. # respectively.
  125. #
  126. # I.e, send/recieve on the connection and its streams are serialized
  127. # separately across the threads accessing the connection. This is a
  128. # simple way of providing thread-safety.
  129. #
  130. # _write_lock and _read_lock synchronize all interactions between
  131. # streams and the connnection. There is a third I/O callback,
  132. # _close_stream, passed to a stream's constructor. It does not need to
  133. # be synchronized, it uses _send_cb internally (which is serialized);
  134. # its other activity (safe deletion of the stream from self.streams)
  135. # does not require synchronization.
  136. #
  137. # _read_lock may be acquired when already holding the _write_lock,
  138. # when they both held it is always by acquiring _write_lock first.
  139. #
  140. # Either _read_lock or _write_lock may be acquired whilst holding _lock
  141. # which should always be acquired before either of the other two.
  142. self._lock = threading.RLock()
  143. self._write_lock = threading.RLock()
  144. self._read_lock = threading.RLock()
  145. # Create the mutable state.
  146. self.__wm_class = window_manager or FlowControlManager
  147. self.__init_state()
  148. return
  149. def __init_state(self):
  150. """
  151. Initializes the 'mutable state' portions of the HTTP/2 connection
  152. object.
  153. This method exists to enable HTTP20Connection objects to be reused if
  154. they're closed, by resetting the connection object to its basic state
  155. whenever it ends up closed. Any situation that needs to recreate the
  156. connection can call this method and it will be done.
  157. This is one of the only methods in hyper that is truly private, as
  158. users should be strongly discouraged from messing about with connection
  159. objects themselves.
  160. """
  161. self._conn = _LockedObject(h2.connection.H2Connection())
  162. # Streams are stored in a dictionary keyed off their stream IDs. We
  163. # also save the most recent one for easy access without having to walk
  164. # the dictionary.
  165. #
  166. # We add a set of all streams that we or the remote party forcefully
  167. # closed with RST_STREAM, to avoid encountering issues where frames
  168. # were already in flight before the RST was processed.
  169. #
  170. # Finally, we add a set of streams that recently received data. When
  171. # using multiple threads, this avoids reading on threads that have just
  172. # acquired the I/O lock whose streams have already had their data read
  173. # for them by prior threads.
  174. self.streams = {}
  175. self.recent_stream = None
  176. self.next_stream_id = 1
  177. self.reset_streams = set()
  178. self.recent_recv_streams = set()
  179. # The socket used to send data.
  180. self._sock = None
  181. # Instantiate a window manager.
  182. self.window_manager = self.__wm_class(65535)
  183. return
  184. def ping(self, opaque_data):
  185. """
  186. Send a PING frame.
  187. Concurrency
  188. -----------
  189. This method is thread-safe.
  190. :param opaque_data: A bytestring of length 8 that will be sent in the
  191. PING frame.
  192. :returns: Nothing
  193. """
  194. self.connect()
  195. with self._write_lock:
  196. with self._conn as conn:
  197. conn.ping(to_bytestring(opaque_data))
  198. self._send_outstanding_data()
  199. def request(self, method, url, body=None, headers=None):
  200. """
  201. This will send a request to the server using the HTTP request method
  202. ``method`` and the selector ``url``. If the ``body`` argument is
  203. present, it should be string or bytes object of data to send after the
  204. headers are finished. Strings are encoded as UTF-8. To use other
  205. encodings, pass a bytes object. The Content-Length header is set to the
  206. length of the body field.
  207. Concurrency
  208. -----------
  209. This method is thread-safe.
  210. :param method: The request method, e.g. ``'GET'``.
  211. :param url: The URL to contact, e.g. ``'/path/segment'``.
  212. :param body: (optional) The request body to send. Must be a bytestring
  213. or a file-like object.
  214. :param headers: (optional) The headers to send on the request.
  215. :returns: A stream ID for the request.
  216. """
  217. headers = headers or {}
  218. # Concurrency
  219. #
  220. # It's necessary to hold a lock while this method runs to satisfy H2
  221. # protocol requirements.
  222. #
  223. # - putrequest obtains the next valid new stream_id
  224. # - endheaders sends a http2 message using the new stream_id
  225. #
  226. # If threads interleave these operations, it could result in messages
  227. # being sent in the wrong order, which can lead to the out-of-order
  228. # messages with lower stream IDs being closed prematurely.
  229. with self._write_lock:
  230. stream_id = self.putrequest(method, url)
  231. default_headers = (':method', ':scheme', ':authority', ':path')
  232. for name, value in headers.items():
  233. is_default = to_native_string(name) in default_headers
  234. self.putheader(name, value, stream_id, replace=is_default)
  235. # Convert the body to bytes if needed.
  236. if body and isinstance(body, (unicode, bytes)):
  237. body = to_bytestring(body)
  238. self.endheaders(message_body=body, final=True, stream_id=stream_id)
  239. return stream_id
  240. def _get_stream(self, stream_id):
  241. if stream_id is None:
  242. return self.recent_stream
  243. elif stream_id in self.reset_streams or stream_id not in self.streams:
  244. raise StreamResetError("Stream forcefully closed")
  245. else:
  246. return self.streams[stream_id]
  247. def get_response(self, stream_id=None):
  248. """
  249. Should be called after a request is sent to get a response from the
  250. server. If sending multiple parallel requests, pass the stream ID of
  251. the request whose response you want. Returns a
  252. :class:`HTTP20Response <hyper.HTTP20Response>` instance.
  253. If you pass no ``stream_id``, you will receive the oldest
  254. :class:`HTTPResponse <hyper.HTTP20Response>` still outstanding.
  255. Concurrency
  256. -----------
  257. This method is thread-safe.
  258. :param stream_id: (optional) The stream ID of the request for which to
  259. get a response.
  260. :returns: A :class:`HTTP20Response <hyper.HTTP20Response>` object.
  261. """
  262. stream = self._get_stream(stream_id)
  263. return HTTP20Response(stream.getheaders(), stream)
  264. def get_pushes(self, stream_id=None, capture_all=False):
  265. """
  266. Returns a generator that yields push promises from the server. **Note
  267. that this method is not idempotent**: promises returned in one call
  268. will not be returned in subsequent calls. Iterating through generators
  269. returned by multiple calls to this method simultaneously results in
  270. undefined behavior.
  271. :param stream_id: (optional) The stream ID of the request for which to
  272. get push promises.
  273. :param capture_all: (optional) If ``False``, the generator will yield
  274. all buffered push promises without blocking. If ``True``, the
  275. generator will first yield all buffered push promises, then yield
  276. additional ones as they arrive, and terminate when the original
  277. stream closes.
  278. :returns: A generator of :class:`HTTP20Push <hyper.HTTP20Push>` objects
  279. corresponding to the streams pushed by the server.
  280. """
  281. stream = self._get_stream(stream_id)
  282. for promised_stream_id, headers in stream.get_pushes(capture_all):
  283. yield HTTP20Push(
  284. HTTPHeaderMap(headers), self.streams[promised_stream_id]
  285. )
  286. def connect(self):
  287. """
  288. Connect to the server specified when the object was created. This is a
  289. no-op if we're already connected.
  290. Concurrency
  291. -----------
  292. This method is thread-safe. It may be called from multiple threads, and
  293. is a noop for all threads apart from the first.
  294. :returns: Nothing.
  295. """
  296. with self._lock:
  297. if self._sock is not None:
  298. return
  299. if not self.proxy_host:
  300. host = self.host
  301. port = self.port
  302. else:
  303. host = self.proxy_host
  304. port = self.proxy_port
  305. sock = socket.create_connection((host, port))
  306. if self.secure:
  307. assert not self.proxy_host, "Proxy with HTTPS not supported."
  308. sock, proto = wrap_socket(sock, host, self.ssl_context,
  309. force_proto=self.force_proto)
  310. else:
  311. proto = H2C_PROTOCOL
  312. log.debug("Selected NPN protocol: %s", proto)
  313. assert proto in H2_NPN_PROTOCOLS or proto == H2C_PROTOCOL
  314. self._sock = BufferedSocket(sock, self.network_buffer_size)
  315. self._send_preamble()
  316. def _connect_upgrade(self, sock):
  317. """
  318. Called by the generic HTTP connection when we're being upgraded. Locks
  319. in a new socket and places the backing state machine into an upgrade
  320. state, then sends the preamble.
  321. """
  322. self._sock = sock
  323. with self._conn as conn:
  324. conn.initiate_upgrade_connection()
  325. conn.update_settings(
  326. {h2.settings.ENABLE_PUSH: int(self._enable_push)}
  327. )
  328. self._send_outstanding_data()
  329. # The server will also send an initial settings frame, so get it.
  330. # However, we need to make sure our stream state is set up properly
  331. # first, or any extra data we receive might cause us problems.
  332. s = self._new_stream(local_closed=True)
  333. self.recent_stream = s
  334. self._recv_cb()
  335. def _send_preamble(self):
  336. """
  337. Sends the necessary HTTP/2 preamble.
  338. """
  339. # We need to send the connection header immediately on this
  340. # connection, followed by an initial settings frame.
  341. with self._conn as conn:
  342. conn.initiate_connection()
  343. conn.update_settings(
  344. {h2.settings.ENABLE_PUSH: int(self._enable_push)}
  345. )
  346. self._send_outstanding_data()
  347. # The server will also send an initial settings frame, so get it.
  348. self._recv_cb()
  349. def close(self, error_code=None):
  350. """
  351. Close the connection to the server.
  352. Concurrency
  353. -----------
  354. This method is thread-safe.
  355. :param error_code: (optional) The error code to reset all streams with.
  356. :returns: Nothing.
  357. """
  358. # Concurrency
  359. #
  360. # It's necessary to hold the lock here to ensure that threads closing
  361. # the connection see consistent state, and to prevent creation of
  362. # of new streams while the connection is being closed.
  363. #
  364. # I/O occurs while the lock is held; waiting threads will see a delay.
  365. with self._lock:
  366. # Close all streams
  367. for stream in list(self.streams.values()):
  368. log.debug("Close stream %d" % stream.stream_id)
  369. stream.close(error_code)
  370. # Send GoAway frame to the server
  371. try:
  372. with self._conn as conn:
  373. conn.close_connection(error_code or 0)
  374. self._send_outstanding_data(tolerate_peer_gone=True)
  375. except Exception as e: # pragma: no cover
  376. log.warn("GoAway frame could not be sent: %s" % e)
  377. if self._sock is not None:
  378. self._sock.close()
  379. self.__init_state()
  380. def _send_outstanding_data(self, tolerate_peer_gone=False,
  381. send_empty=True):
  382. # Concurrency
  383. #
  384. # Hold _write_lock; getting and writing data from _conn is synchronized
  385. #
  386. # I/O occurs while the lock is held; waiting threads will see a delay.
  387. with self._write_lock:
  388. with self._conn as conn:
  389. data = conn.data_to_send()
  390. if data or send_empty:
  391. self._send_cb(data, tolerate_peer_gone=tolerate_peer_gone)
  392. def putrequest(self, method, selector, **kwargs):
  393. """
  394. This should be the first call for sending a given HTTP request to a
  395. server. It returns a stream ID for the given connection that should be
  396. passed to all subsequent request building calls.
  397. Concurrency
  398. -----------
  399. This method is thread-safe. It can be called from multiple threads,
  400. and each thread should receive a unique stream ID.
  401. :param method: The request method, e.g. ``'GET'``.
  402. :param selector: The path selector.
  403. :returns: A stream ID for the request.
  404. """
  405. # Create a new stream.
  406. s = self._new_stream()
  407. # To this stream we need to immediately add a few headers that are
  408. # HTTP/2 specific. These are: ":method", ":scheme", ":authority" and
  409. # ":path". We can set all of these now.
  410. s.add_header(":method", method)
  411. s.add_header(":scheme", "https" if self.secure else "http")
  412. s.add_header(":authority", self.host)
  413. s.add_header(":path", selector)
  414. # Save the stream.
  415. self.recent_stream = s
  416. return s.stream_id
  417. def putheader(self, header, argument, stream_id=None, replace=False):
  418. """
  419. Sends an HTTP header to the server, with name ``header`` and value
  420. ``argument``.
  421. Unlike the ``httplib`` version of this function, this version does not
  422. actually send anything when called. Instead, it queues the headers up
  423. to be sent when you call
  424. :meth:`endheaders() <hyper.HTTP20Connection.endheaders>`.
  425. This method ensures that headers conform to the HTTP/2 specification.
  426. In particular, it strips out the ``Connection`` header, as that header
  427. is no longer valid in HTTP/2. This is to make it easy to write code
  428. that runs correctly in both HTTP/1.1 and HTTP/2.
  429. :param header: The name of the header.
  430. :param argument: The value of the header.
  431. :param stream_id: (optional) The stream ID of the request to add the
  432. header to.
  433. :returns: Nothing.
  434. """
  435. stream = self._get_stream(stream_id)
  436. stream.add_header(header, argument, replace)
  437. return
  438. def endheaders(self, message_body=None, final=False, stream_id=None):
  439. """
  440. Sends the prepared headers to the server. If the ``message_body``
  441. argument is provided it will also be sent to the server as the body of
  442. the request, and the stream will immediately be closed. If the
  443. ``final`` argument is set to True, the stream will also immediately
  444. be closed: otherwise, the stream will be left open and subsequent calls
  445. to ``send()`` will be required.
  446. :param message_body: (optional) The body to send. May not be provided
  447. assuming that ``send()`` will be called.
  448. :param final: (optional) If the ``message_body`` parameter is provided,
  449. should be set to ``True`` if no further data will be provided via
  450. calls to :meth:`send() <hyper.HTTP20Connection.send>`.
  451. :param stream_id: (optional) The stream ID of the request to finish
  452. sending the headers on.
  453. :returns: Nothing.
  454. """
  455. self.connect()
  456. stream = self._get_stream(stream_id)
  457. headers_only = (message_body is None and final)
  458. # Concurrency:
  459. #
  460. # Hold _write_lock: synchronize access to the connection's HPACK
  461. # encoder and decoder and the subsquent write to the connection
  462. with self._write_lock:
  463. stream.send_headers(headers_only)
  464. # Send whatever data we have.
  465. if message_body is not None:
  466. stream.send_data(message_body, final)
  467. self._send_outstanding_data()
  468. return
  469. def send(self, data, final=False, stream_id=None):
  470. """
  471. Sends some data to the server. This data will be sent immediately
  472. (excluding the normal HTTP/2 flow control rules). If this is the last
  473. data that will be sent as part of this request, the ``final`` argument
  474. should be set to ``True``. This will cause the stream to be closed.
  475. :param data: The data to send.
  476. :param final: (optional) Whether this is the last bit of data to be
  477. sent on this request.
  478. :param stream_id: (optional) The stream ID of the request to send the
  479. data on.
  480. :returns: Nothing.
  481. """
  482. stream = self._get_stream(stream_id)
  483. stream.send_data(data, final)
  484. return
  485. def _new_stream(self, stream_id=None, local_closed=False):
  486. """
  487. Returns a new stream object for this connection.
  488. """
  489. # Concurrency
  490. #
  491. # Hold _lock: ensure that threads accessing the connection see
  492. # self.next_stream_id in a consistent state
  493. #
  494. # No I/O occurs, the delay in waiting threads depends on their number.
  495. with self._lock:
  496. s = Stream(
  497. stream_id or self.next_stream_id,
  498. self.__wm_class(DEFAULT_WINDOW_SIZE),
  499. self._conn,
  500. self._send_outstanding_data,
  501. self._recv_cb,
  502. self._stream_close_cb,
  503. )
  504. s.local_closed = local_closed
  505. self.streams[s.stream_id] = s
  506. self.next_stream_id += 2
  507. return s
  508. def _send_cb(self, data, tolerate_peer_gone=False):
  509. """
  510. This is the callback used by streams to send data on the connection.
  511. This acts as a dumb wrapper around the socket send method.
  512. """
  513. # Concurrency
  514. #
  515. # Hold _write_lock: ensures only writer at a time
  516. #
  517. # I/O occurs while the lock is held; waiting threads will see a delay.
  518. with self._write_lock:
  519. try:
  520. self._sock.sendall(data)
  521. except socket.error as e:
  522. if (not tolerate_peer_gone or
  523. e.errno not in (errno.EPIPE, errno.ECONNRESET)):
  524. raise
  525. def _adjust_receive_window(self, frame_len):
  526. """
  527. Adjusts the window size in response to receiving a DATA frame of length
  528. ``frame_len``. May send a WINDOWUPDATE frame if necessary.
  529. """
  530. # Concurrency
  531. #
  532. # Hold _write_lock; synchronize the window manager update and the
  533. # subsequent potential write to the connection
  534. #
  535. # I/O may occur while the lock is held; waiting threads may see a
  536. # delay.
  537. with self._write_lock:
  538. increment = self.window_manager._handle_frame(frame_len)
  539. if increment:
  540. with self._conn as conn:
  541. conn.increment_flow_control_window(increment)
  542. self._send_outstanding_data(tolerate_peer_gone=True)
  543. return
  544. def _single_read(self):
  545. """
  546. Performs a single read from the socket and hands the data off to the
  547. h2 connection object.
  548. """
  549. # Begin by reading what we can from the socket.
  550. #
  551. # Concurrency
  552. #
  553. # Synchronizes reading the data
  554. #
  555. # I/O occurs while the lock is held; waiting threads will see a delay.
  556. with self._read_lock:
  557. if self._sock is None:
  558. raise ConnectionError('tried to read after connection close')
  559. self._sock.fill()
  560. data = self._sock.buffer.tobytes()
  561. self._sock.advance_buffer(len(data))
  562. with self._conn as conn:
  563. events = conn.receive_data(data)
  564. stream_ids = set(getattr(e, 'stream_id', -1) for e in events)
  565. stream_ids.discard(-1) # sentinel
  566. stream_ids.discard(0) # connection events
  567. self.recent_recv_streams |= stream_ids
  568. for event in events:
  569. if isinstance(event, h2.events.DataReceived):
  570. self._adjust_receive_window(event.flow_controlled_length)
  571. self.streams[event.stream_id].receive_data(event)
  572. elif isinstance(event, h2.events.PushedStreamReceived):
  573. if self._enable_push:
  574. self._new_stream(event.pushed_stream_id, local_closed=True)
  575. self.streams[event.parent_stream_id].receive_push(event)
  576. else:
  577. # Servers are forbidden from sending push promises when
  578. # the ENABLE_PUSH setting is 0, but the spec leaves the
  579. # client action undefined when they do it anyway. So we
  580. # just refuse the stream and go about our business.
  581. self._send_rst_frame(event.pushed_stream_id, 7)
  582. elif isinstance(event, h2.events.ResponseReceived):
  583. self.streams[event.stream_id].receive_response(event)
  584. elif isinstance(event, h2.events.TrailersReceived):
  585. self.streams[event.stream_id].receive_trailers(event)
  586. elif isinstance(event, h2.events.StreamEnded):
  587. self.streams[event.stream_id].receive_end_stream(event)
  588. elif isinstance(event, h2.events.StreamReset):
  589. if event.stream_id not in self.reset_streams:
  590. self.reset_streams.add(event.stream_id)
  591. self.streams[event.stream_id].receive_reset(event)
  592. elif isinstance(event, h2.events.ConnectionTerminated):
  593. # If we get GoAway with error code zero, we are doing a
  594. # graceful shutdown and all is well. Otherwise, throw an
  595. # exception.
  596. self.close()
  597. # If an error occured, try to read the error description from
  598. # code registry otherwise use the frame's additional data.
  599. if event.error_code != 0:
  600. try:
  601. name, number, description = errors.get_data(
  602. event.error_code
  603. )
  604. except ValueError:
  605. error_string = (
  606. "Encountered error code %d" % event.error_code
  607. )
  608. else:
  609. error_string = (
  610. "Encountered error %s %s: %s" %
  611. (name, number, description)
  612. )
  613. raise ConnectionError(error_string)
  614. else:
  615. log.info("Received unhandled event %s", event)
  616. self._send_outstanding_data(tolerate_peer_gone=True, send_empty=False)
  617. def _recv_cb(self, stream_id=0):
  618. """
  619. This is the callback used by streams to read data from the connection.
  620. This stream reads what data it can, and throws it into the underlying
  621. connection, before farming out any events that fire to the relevant
  622. streams. If the socket remains readable, it will then optimistically
  623. continue to attempt to read.
  624. This is generally called by a stream, not by the connection itself, and
  625. it's likely that streams will read a frame that doesn't belong to them.
  626. :param stream_id: (optional) The stream ID of the stream reading data
  627. from the connection.
  628. """
  629. # Begin by reading what we can from the socket.
  630. #
  631. # Concurrency
  632. #
  633. # Ignore this read if some other thread has recently read data from
  634. # from the requested stream.
  635. #
  636. # The lock here looks broad, but is needed to ensure correct behavior
  637. # when there are multiple readers of the same stream. It is
  638. # re-acquired in the calls to self._single_read.
  639. #
  640. # I/O occurs while the lock is held; waiting threads will see a delay.
  641. with self._read_lock:
  642. log.debug('recv for stream %d with %s already present',
  643. stream_id,
  644. self.recent_recv_streams)
  645. if stream_id in self.recent_recv_streams:
  646. self.recent_recv_streams.discard(stream_id)
  647. return
  648. # make sure to validate the stream is readable.
  649. # if the connection was reset, this stream id won't appear in
  650. # self.streams and will cause this call to raise an exception.
  651. if stream_id:
  652. self._get_stream(stream_id)
  653. # TODO: Re-evaluate this.
  654. self._single_read()
  655. count = 9
  656. retry_wait = 0.05 # can improve responsiveness to delay the retry
  657. while count and self._sock is not None and self._sock.can_read:
  658. # If the connection has been closed, bail out, but retry
  659. # on transient errors.
  660. try:
  661. self._single_read()
  662. except ConnectionResetError:
  663. break
  664. except ssl.SSLError as e: # pragma: no cover
  665. # these are transient errors that can occur while reading
  666. # from ssl connections.
  667. if e.args[0] in TRANSIENT_SSL_ERRORS:
  668. continue
  669. else:
  670. raise
  671. except socket.error as e: # pragma: no cover
  672. if e.errno in (errno.EINTR, errno.EAGAIN):
  673. # if 'interrupted' or 'try again', continue
  674. time.sleep(retry_wait)
  675. continue
  676. elif e.errno == errno.ECONNRESET:
  677. break
  678. else:
  679. raise
  680. count -= 1
  681. def _send_rst_frame(self, stream_id, error_code):
  682. """
  683. Send reset stream frame with error code and remove stream from map.
  684. """
  685. # Concurrency
  686. #
  687. # Hold _write_lock; synchronize generating the reset frame and writing
  688. # it
  689. #
  690. # I/O occurs while the lock is held; waiting threads will see a delay.
  691. with self._write_lock:
  692. with self._conn as conn:
  693. conn.reset_stream(stream_id, error_code=error_code)
  694. self._send_outstanding_data()
  695. # Concurrency
  696. #
  697. # Hold _lock; the stream storage is being updated. No I/O occurs, any
  698. # delay is proportional to the number of waiting threads.
  699. with self._lock:
  700. try:
  701. del self.streams[stream_id]
  702. self.recent_recv_streams.discard(stream_id)
  703. except KeyError as e: # pragma: no cover
  704. log.warn(
  705. "Stream with id %d does not exist: %s",
  706. stream_id, e)
  707. # Keep track of the fact that we reset this stream in case there
  708. # are other frames in flight.
  709. self.reset_streams.add(stream_id)
  710. def _stream_close_cb(self, stream_id):
  711. """
  712. Called by a stream when it is closing, so that state can be cleared.
  713. """
  714. try:
  715. del self.streams[stream_id]
  716. self.recent_recv_streams.discard(stream_id)
  717. except KeyError:
  718. pass
  719. # The following two methods are the implementation of the context manager
  720. # protocol.
  721. def __enter__(self):
  722. return self
  723. def __exit__(self, type, value, tb):
  724. self.close()
  725. return False # Never swallow exceptions.