我们从Python开源项目中，提取了以下23个代码示例，用于说明如何使用pycurl.CurlMulti()。
def __new__(cls, io_loop=None, max_clients=10, max_simultaneous_connections=None): # There is one client per IOLoop since they share curl instances io_loop = io_loop or ioloop.IOLoop.instance() if io_loop in cls._ASYNC_CLIENTS: return cls._ASYNC_CLIENTS[io_loop] else: instance = super(AsyncHTTPClient, cls).__new__(cls) instance.io_loop = io_loop instance._multi = pycurl.CurlMulti() instance._curls = [_curl_create(max_simultaneous_connections) for i in xrange(max_clients)] instance._free_list = instance._curls[:] instance._requests = collections.deque() instance._fds = {} instance._events = {} instance._added_perform_callback = False instance._timeout = None instance._closed = False cls._ASYNC_CLIENTS[io_loop] = instance return instance
def __new__(cls, io_loop=None, max_clients=10, max_simultaneous_connections=None): # There is one client per IOLoop since they share curl instances io_loop = io_loop or ioloop.IOLoop.instance() if io_loop in cls._ASYNC_CLIENTS: return cls._ASYNC_CLIENTS[io_loop] else: instance = super(AsyncHTTPClient2, cls).__new__(cls) instance.io_loop = io_loop instance._multi = pycurl.CurlMulti() instance._multi.setopt(pycurl.M_TIMERFUNCTION, instance._set_timeout) instance._multi.setopt(pycurl.M_SOCKETFUNCTION, instance._handle_socket) instance._curls = [_curl_create(max_simultaneous_connections) for i in xrange(max_clients)] instance._free_list = instance._curls[:] instance._requests = collections.deque() instance._fds = {} instance._timeout = None cls._ASYNC_CLIENTS[io_loop] = instance return instance
def initialize(self, io_loop, max_clients=10, defaults=None): super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [self._curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl # multi object are initialized lazily, and its destructor will # segfault if it is destroyed without having been used. Add # and remove a dummy handle to make sure everything is # initialized. dummy_curl_handle = pycurl.Curl() self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle)
def __init__(self, multi_handler=None, max_simultanous_connections=None, defer_callbacks=False): self.requests = {} if multi_handler: self._curl_multi_handler = multi_handler else: self._curl_multi_handler = pycurl.CurlMulti() if max_simultanous_connections is not None: self._curl_multi_handler.setopt(pycurl.M_MAX_TOTAL_CONNECTIONS, max_simultanous_connections) self._defer_callbacks = defer_callbacks self._async_callbacks = [] self._async_results = []
def create_pool(self, num_conn): # Pre-allocate a list of curl objects self.m = pycurl.CurlMulti() self.m.handles = [] for i in range(num_conn): c = pycurl.Curl() self.m.handles.append(c) self.freelist.put(c)
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, progressNotify=None, disposition=False): self.url = url self.filename = filename #complete file destination, not only name self.get = get self.post = post self.referer = referer self.cj = cj #cookiejar if cookies are needed self.bucket = bucket self.options = options self.disposition = disposition # all arguments self.abort = False self.size = 0 self.nameDisposition = None #will be parsed from content disposition self.chunks = [] self.log = getLogger("log") try: self.info = ChunkInfo.load(filename) self.info.resume = True #resume is only possible with valid info file self.size = self.info.size self.infoSaved = True except IOError: self.info = ChunkInfo(filename) self.chunkSupport = None self.m = pycurl.CurlMulti() #needed for speed calculation self.lastArrived = [] self.speeds = [] self.lastSpeeds = [0, 0] self.progressNotify = progressNotify
def __init__(self, p_timeoutMs=1000, p_curlMOpts=None): self.m_handle = pycurl.CurlMulti() self.m_clients = [] self.m_timeoutMs = p_timeoutMs self.m_opts = p_curlMOpts if p_curlMOpts is None: self.m_opts = {} self._init_opt()
def initialize(self, io_loop=None, max_clients=10, max_simultaneous_connections=None): self.io_loop = io_loop self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [_curl_create(max_simultaneous_connections) for i in xrange(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None try: self._socket_action = self._multi.socket_action except AttributeError: # socket_action is found in pycurl since 7.18.2 (it's been # in libcurl longer than that but wasn't accessible to # python). logging.warning("socket_action method missing from pycurl; " "falling back to socket_all. Upgrading " "libcurl and pycurl will improve performance") self._socket_action = \ lambda fd, action: self._multi.socket_all() # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start()
def initialize(self, io_loop, max_clients=10, defaults=None): super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [_curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} self._timeout = None try: self._socket_action = self._multi.socket_action except AttributeError: # socket_action is found in pycurl since 7.18.2 (it's been # in libcurl longer than that but wasn't accessible to # python). gen_log.warning("socket_action method missing from pycurl; " "falling back to socket_all. Upgrading " "libcurl and pycurl will improve performance") self._socket_action = \ lambda fd, action: self._multi.socket_all() # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl # multi object are initialized lazily, and its destructor will # segfault if it is destroyed without having been used. Add # and remove a dummy handle to make sure everything is # initialized. dummy_curl_handle = pycurl.Curl() self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle)
def __init__(self, *args, **kwargs): DownloadRequest.__init__(self, *args, **kwargs) self.path = None self.disposition = False self.chunks = [] self.chunk_support = None self.__manager = pycurl.CurlMulti() # needed for speed calculation self.last_arrived = [] self.speeds = [] self.last_speeds = [0, 0]
def __init__(self, start_id, conns=2, skip_timeout=120): Thread.__init__(self) self.m = pycurl.CurlMulti() self.handles = [self._create_handle() for i in range(conns)] self.free_handles = list(self.handles) self.last_request = None self.skip_ahead = False self.skip_timeout = skip_timeout self.dl_deltas = deque(maxlen=10) self.req_time = deque(maxlen=20) self.queue_time = deque(maxlen=20) self.peek_time = deque(maxlen=20) self.req_delay_time = deque(maxlen=20) self.stat_thread = Thread(target=self.calc_stat) self.dps = deque(maxlen=60) self.delta_lock = Lock() self.delta_count = 0 # queue for requests so we can ensure a delay between each request # use a list instead of an actual queue to be able to choose min ID self.req_queue = [] self.req_queue_lock = Lock() # requests by order self.requests = [] self.requests_lock = Lock() self.res_queue = Queue(maxsize=RES_QUEUE_MAXSIZE) self.evt_stop = Event() self.add_request(start_id)
def __init__(self, transport, max_conn=20): # Backpointer to transport object self.__xport = transport # Curl handles self.__mhandle = pycurl.CurlMulti() self.__chandles = [] self.__active_handles = 0 self.__max_handles = max_conn # Request queue self.__req_q = deque() # List of failures self.__failures = [] # List of URLs successfully transferred self.__success = [] # List of Orphaned URLs. self.__orphans = set() # Set default file buffer size at 128k, callers override # this setting after looking at VFS block size. self.__file_bufsz = 131072 # Header bits and pieces self.__user_agent = None self.__common_header = {} self.__last_stall_check = 0 # Set options on multi-handle self.__mhandle.setopt(pycurl.M_PIPELINING, 0) # initialize easy handles for i in range(self.__max_handles): eh = pycurl.Curl() eh.url = None eh.repourl = None eh.fobj = None eh.r_fobj = None eh.filepath = None eh.success = False eh.fileprog = None eh.filetime = -1 eh.starttime = -1 eh.uuid = None self.__chandles.append(eh) # copy handles into handle freelist self.__freehandles = self.__chandles[:]
