Python aiohttp 模块，ServerDisconnectedError() 实例源码

def test_single_proxy(self, proxy):
        """
        text one proxy, if valid, put them to usable_proxies.
        """
        try:
            async with aiohttp.ClientSession() as session:
                try:
                    if isinstance(proxy, bytes):
                        proxy = proxy.decode('utf-8')
                    real_proxy = 'http://' + proxy
                    print('Testing', proxy)
                    async with session.get(self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) as response:
                        if response.status == 200:
                            self._conn.put(proxy)
                            print('Valid proxy', proxy)
                except (ProxyConnectionError, TimeoutError, ValueError):
                    print('Invalid proxy', proxy)
        except (ServerDisconnectedError, ClientResponseError,ClientConnectorError) as s:
            print(s)
            pass

def read_stream(app_name, auth_token):
    while True:
        stream_url = yield from get_stream_url(app_name, auth_token)
        print('Reading stream: %s' % stream_url)
        log = b''
        with aiohttp.ClientSession() as session:
            response = yield from session.get(stream_url)
            while True:
                try:
                    chunk = yield from response.content.read(1)
                except aiohttp.ServerDisconnectedError:
                    break
                if not chunk:
                    break
                if chunk == b'\n':
                    try:
                        yield from write_to_queue(log)
                    except ValueError as e:
                        print(str(e))
                    log = b''
                else:
                    log += chunk

def target_fetch(client, url):
    """
    :param client: aiohttp client
    :param url: target url
    :return: text
    """
    with async_timeout.timeout(30):
        try:
            headers = {'user-agent': get_random_user_agent()}
            async with client.get(url, headers=headers) as response:
                assert response.status == 200
                LOGGER.info('Task url: {}'.format(response.url))
                try:
                    text = await response.text()
                except:
                    try:
                        text = await response.read()
                    except aiohttp.ServerDisconnectedError as e:
                        LOGGER.exception(e)
                        text = None
                return text
        except Exception as e:
            LOGGER.exception(e)
            return None

def longpoll_processor(self):
        await self.init_long_polling()

        session = aiohttp.ClientSession(loop=self.loop)

        while True:
            try:
                self.longpoll_request = session.get(self.server, params=self.values)

                resp = await self.longpoll_request

            except aiohttp.ClientOSError:
                session = aiohttp.ClientSession(loop=self.loop)

            except (asyncio.TimeoutError, aiohttp.ServerDisconnectedError):
                self.logger.warning("Long polling server doesn't respond. Changing server")
                await self.init_long_polling()
                continue

            try:
                events = json.loads(await resp.text())
            except ValueError:
                continue

            failed = events.get('failed')

            if failed:
                err_num = int(failed)

                if err_num == 1:  # 1 - update timestamp
                    self.values['ts'] = events['ts']

                elif err_num in (2, 3):  # 2, 3 - new data for long polling
                    await self.init_long_polling(err_num)

                continue

            self.values['ts'] = events['ts']
            for event in events['updates']:
                asyncio.ensure_future(self.process_longpoll_event(event))

def clear_channel(self, channel):
        current_index = None
        last_index = None
        num_errors = 0

        while current_index != -1:
            if num_errors >= 5:
                break

            try:
                async for message in self.logs_from(channel, before=current_index):
                    current_index = message
                    await self.delete_message(message)
            except ServerDisconnectedError:
                try:
                    async for message in self.logs_from(channel, before=current_index):
                        current_index = message
                        await self.delete_message(message)
                except Exception:
                    num_errors += 1
                    continue
            except Exception:
                num_errors += 1
                continue

            if last_index == current_index:
                break

            last_index = current_index

def test_disconnect(self, mock_book, mock_connect):
        mock_connect.return_value.aenter.receive_str = CoroutineMock()
        mock_connect.return_value.aenter.send_json = CoroutineMock()
        mock_book.return_value = {'bids': [], 'asks': [], 'sequence': 1}

        messages_expected = [
            json.dumps({
              "type": "done",
              "side": "sell",
              "order_id": "4eef1226-4b38-422c-a5b1-56def7107f9a",
              "reason": "canceled",
              "product_id": "ETH-USD",
              "price": "2601.76000000",
              "remaining_size": "3.09000000",
              "sequence": 2,
              "time": "2017-06-25T11:23:14.775000Z"
            }),
            aiohttp.ServerDisconnectedError('error'),
            json.dumps({
              "type": "done",
              "side": "sell",
              "order_id": "4eef1226-4b38-422c-a5b1-56def7107f9a",
              "reason": "canceled",
              "product_id": "ETH-USD",
              "price": "2601.76000000",
              "remaining_size": "3.09000000",
              "sequence": 2,
              "time": "2017-06-25T11:23:14.775000Z"
            })
        ]
        mock_connect.return_value.aenter.receive_str.side_effect = \
            messages_expected
        async with gdax.orderbook.OrderBook() as orderbook:
            message = await orderbook.handle_message()
            assert message == json.loads(messages_expected[0])

            message = await orderbook.handle_message()
            assert message is None

            message = await orderbook.handle_message()
            assert message == json.loads(messages_expected[2])

def download(self) -> str:
        if self._state != NOT_STARTED:
            return self._state

        # Prepare an empty buffer file.
        await self._loop.run_in_executor(None, self._create_buffer_file)

        try:
            # Create client session for downloading a file part from a host.
            async with aiohttp.ClientSession(loop=self._loop, headers=self._headers) as session:
                # Request a host for a file part.
                async with session.request('GET', self._url) as res:  # type: aiohttp.ClientResponse
                    if res.status != 206:
                        raise WrongResponseError('Expected status code 206, but {!s} ({!s}) received.',
                                                 res.status,
                                                 res.reason)

                    hrh = res.headers  # type: CIMultiDictProxy
                    # TODO: check headers.

                    # Read content by chunks and write to the buffer file.
                    if self._state == NOT_STARTED:
                        self._state = DOWNLOADING
                    while self._state is DOWNLOADING:
                        with aiohttp.Timeout(self._chunk_download_timeout, loop=self._loop):
                            chunk = await res.content.read(self._chunk_size)
                            self._bytes_downloaded += len(chunk)

                            self._debug("Read ({!s} bytes). Downloaded: {!s} of {!s} bytes. [{:.2%}]".format(
                                len(chunk), self._bytes_downloaded, self._length,
                                self._bytes_downloaded / self._length))

                            if not chunk:
                                self._state = DOWNLOADED
                                break
                            await self._write_chunk(chunk)
                    await self._flush_and_release()
                    if self._state != DOWNLOADED:
                        res.close()  # Close the response if not downloaded.
        except aiohttp.ServerDisconnectedError as exc:
            self._debug('Server disconnected error: {!r}.'.format(exc))
            self.cancel()
        except WrongResponseError as exc:
            self._debug('Wrong response error: {!r}.'.format(exc))
            self.cancel()
        except asyncio.TimeoutError:
            self._debug('Timeout.')
            self.cancel()
        except Exception as exc:
            self._debug('Unexpected exception: {!r}.'.format(exc))
            self.cancel()
        finally:
            return self._state

def update_user_data():
    """ Go through all registered members playing osu!, and update their data. """
    global osu_tracking

    # Go through each member playing and give them an "old" and a "new" subsection
    # for their previous and latest user data
    for member_id, profile in osu_config.data["profiles"].items():
        member = discord.utils.find(lambda m: check_playing(m, member_id), client.get_all_members())

        # If the member is not playing anymore, remove them from the tracking data
        if not member:
            if member_id in osu_tracking:
                del osu_tracking[member_id]

            continue

        mode = get_mode(member_id).value
        try:
            user_data = await api.get_user(u=profile, type="id", m=mode)
        except ServerDisconnectedError:
            continue
        except asyncio.TimeoutError:
            logging.warning("Timed out when retrieving osu! info from {} ({})".format(member, profile))
            continue

        # Sleep after using get_user as to not put too much strain on the API at once
        await asyncio.sleep(.2)

        # Just in case something goes wrong, we skip this member (these things are usually one-time occurrences)
        if user_data is None:
            logging.info("Could not retrieve osu! info from {} ({})".format(member, profile))
            continue

        # User is already tracked
        if member_id in osu_tracking:
            # Move the "new" data into the "old" data of this user
            osu_tracking[member_id]["old"] = osu_tracking[member_id]["new"]
        else:
            # If this is the first time, update the user's list of scores for later
            user_scores = await api.get_user_best(u=profile, type="id", limit=score_request_limit, m=mode)
            osu_tracking[member_id] = dict(member=member, scores=user_scores)

        # Update the "new" data
        osu_tracking[member_id]["new"] = user_data
        osu_tracking[member_id]["new"]["ripple"] = True if api.ripple_pattern.match(profile) else False

def _send(self, payload, processor):
        '''Send a payload to be converted to JSON.

        Handles temporary connection issues.  Daemon reponse errors
        are raise through DaemonError.
        '''
        def log_error(error):
            self.down = True
            now = time.time()
            prior_time = self.last_error_time
            if now - prior_time > 60:
                self.last_error_time = now
                if prior_time and self.failover():
                    secs = 0
                else:
                    self.logger.error('{}  Retrying occasionally...'
                                      .format(error))

        data = json.dumps(payload)
        secs = 1
        max_secs = 4
        while True:
            try:
                result = await self._send_data(data)
                if not isinstance(result, tuple):
                    result = processor(result)
                    if self.down:
                        self.down = False
                        self.last_error_time = 0
                        self.logger.info('connection restored')
                    return result
                log_error('HTTP error code {:d}: {}'
                          .format(result[0], result[1]))
            except asyncio.TimeoutError:
                log_error('timeout error.')
            except aiohttp.ServerDisconnectedError:
                log_error('disconnected.')
            except self.ClientHttpProcessingError:
                log_error('HTTP error.')
            except self.ClientPayloadError:
                log_error('payload encoding error.')
            except aiohttp.ClientConnectionError:
                log_error('connection problem - is your daemon running?')
            except self.DaemonWarmingUpError:
                log_error('starting up checking blocks.')
            except (asyncio.CancelledError, DaemonError):
                raise
            except Exception:
                self.log_error(traceback.format_exc())

            await asyncio.sleep(secs)
            secs = min(max_secs, secs * 2, 1)

def handle_message(self):
        try:
            message = await self._recv()
        except aiohttp.ServerDisconnectedError as exc:
            logging.error(
                f'Error: Exception: f{exc}. Re-initializing websocket.')
            await self.__aexit__(None, None, None)
            await self.__aenter__()
            return

        msg_type = message['type']

        if msg_type == 'error':
            raise OrderBookError(f'Error: {message["message"]}')

        if msg_type == 'subscriptions':
            return  # must filter out here because the subscriptions message does not have a product_id key

        product_id = message['product_id']
        assert self._sequences[product_id] is not None
        sequence = message['sequence']

        if sequence <= self._sequences[product_id]:
            # ignore older messages (e.g. before order book initialization
            # from getProductOrderBook)
            return message
        elif sequence > self._sequences[product_id] + 1:
            logging.error(
                'Error: messages missing ({} - {}). Re-initializing websocket.'
                .format(sequence, self._sequences[product_id]))
            await self.__aexit__(None, None, None)
            await self.__aenter__()
            return

        if msg_type == 'open':
            self.add(product_id, message)
        elif msg_type == 'done' and 'price' in message:
            self.remove(product_id, message)
        elif msg_type == 'match':
            self.match(product_id, message)
        elif msg_type == 'change':
            self.change(product_id, message)
        elif msg_type == 'heartbeat':
            pass
        elif msg_type == 'received':
            pass
        elif msg_type == 'done':
            pass
        else:
            raise OrderBookError(f'unknown message type {msg_type}')

        self._sequences[product_id] = sequence
        return message