Python progressbar 模块，FileTransferSpeed() 实例源码

def main():
    uri, outfile, dataset = get_arguments()
    fd = tempfile.NamedTemporaryFile()
    progress = ProgressBar(widgets=[Percentage(), ' ', Bar(), ' ', ETA(), ' ', FileTransferSpeed()])

    def update(count, blockSize, totalSize):
        if progress.maxval is None:
            progress.maxval = totalSize
            progress.start()
        progress.update(min(count * blockSize, totalSize))

    urllib.urlretrieve(uri, fd.name, reporthook = update)
    if dataset == 'zinc12':
        df = pandas.read_csv(fd.name, delimiter = '\t')
        df = df.rename(columns={'SMILES':'structure'})
        df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
    elif dataset == 'chembl22':
        df = pandas.read_table(fd.name,compression='gzip')
        df = df.rename(columns={'canonical_smiles':'structure'})
        df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
        pass
    else:
        df = pandas.read_csv(fd.name, delimiter = '\t')
        df.to_hdf(outfile, 'table', format = 'table', data_columns = True)

def getProgress(self, url, fileSize):
        status = json.loads(urllib.urlopen(url).read())
        if len(status["data"]) ==0 :
            logger.info(url + " upload done ")
            return  True
        widgets = ['Progress: ', Percentage(), ' ', Bar(
            marker=RotatingMarker('>-=')), ' ', ETA(), ' ', FileTransferSpeed()]
        pbar = ProgressBar(widgets=widgets, maxval=fileSize).start()
        upload_size = 0
        while upload_size < fileSize:
            _response = self.doGet(url)
            _data = json.loads(_response)
            upload_size = long(_data["data"]["upload_size"])
            total_size = long(_data["data"]["total_size"])
            if upload_size == 0 and total_size == 0:
                break
            pbar.update(upload_size)
            time.sleep(1)
        pbar.finish()
        logger.info(url + " upload done")
        return True

        """
        ??????
        """

def download(number, save_dir='./'):
    """Download pre-trained word vector
    :param number: integer, default ``None``
    :param save_dir: str, default './'
    :return: file path for downloaded file
    """
    df = load_datasets()

    row = df.iloc[[number]]
    url = ''.join(row.URL)
    if not url:
        print('The word vector you specified was not found. Please specify correct name.')

    widgets = ['Test: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA(), ' ', FileTransferSpeed()]
    pbar = ProgressBar(widgets=widgets)

    def dlProgress(count, blockSize, totalSize):
        if pbar.max_value is None:
            pbar.max_value = totalSize
            pbar.start()

        pbar.update(min(count * blockSize, totalSize))

    file_name = url.split('/')[-1]
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    save_path = os.path.join(save_dir, file_name)
    path, _ = urlretrieve(url, save_path, reporthook=dlProgress)
    pbar.finish()
    return path

def __enter__(self):
        self.bar = progressbar.ProgressBar(
            widgets=[
                progressbar.Percentage(),
                ' ',
                progressbar.Bar(),
                progressbar.FileTransferSpeed(),
                ' ',
                progressbar.ETA(),
            ],
            max_value=self.max_value,
        )
        self.fd = open(self.output_path, 'wb')
        return self

def __init__(self, nbytes, nfiles):
    self._total_bytes = nbytes
    self._pending_files = nfiles
    self._transferring_files = 0
    self._complete_files = 0
    self._lock = Lock()
    self._data = {}

    widgets = ['Progress: ', Percentage(), ' ', Bar(left='[',right=']'),
               ' ', Timer(format='Time: %s'), ' ', FileTransferSpeed()]

    if self._total_bytes > 0:
      self.pbar = ProgressBar(widgets=widgets, maxval=self._total_bytes).start()
    else:
      self.pbar = ProgressBar(widgets=widgets, maxval=nfiles).start()

def setup_progressbar(self):
        from progressbar import ProgressBar, FileTransferSpeed, Bar, Percentage, ETA
        return ProgressBar(widgets=[FileTransferSpeed(), ' <<<', Bar(), '>>> ', Percentage(), ' ', ETA()])

def doUploadFileProgress(self,filePath, url):
        startTime = getNow()
        result = False
        try:
            widgets = ['Progress: ', Percentage(), ' ', Bar(
                marker=RotatingMarker('>-=')), ' ', ETA(), ' ', FileTransferSpeed()]
            pbar = ProgressBar(widgets=widgets, maxval=os.path.getsize(filePath)).start()
            progress = Progress()
            fileSizeStr = formatSize(os.path.getsize(filePath))
            logger.info("??????{0} ?? {1}".format(filePath,fileSizeStr))
            stream = file_with_callback(filePath, 'rb', progress.update,os.path.basename(filePath),pbar)
            params = {"filedata": stream}
            datagen, headers = multipart_encode(params)
            upload_request =urllib2.Request(url, datagen, headers)
            response = urllib2.urlopen(upload_request).read()
            endTime = getNow()
            totlaTime = caltime(startTime, endTime)
            logger.info("??????{0} ????{1} ????{2} ????{3} ??{4} ????{5}"
                        .format(filePath,startTime, endTime, fileSizeStr, totlaTime,response))
            #???????????'4b ? 0'???json ??errmsg("Extra data", s, end, len(s) ??????,??????????
            if "code\":0" in response.replace(' ', ''):
                result = True
            else:
                result = json.loads(response)["code"] == 0
        except Exception as e:
            logger.error("??????{0} exception: {1}".format(filePath,e))
        return result

def _addresses_to_check_with_caching(self, show_progress=True):
        num_addrs = len(list(self._addresses_to_check()))
        widgets = ['ROP: ', progressbar.Percentage(), ' ',
                   progressbar.Bar(marker=progressbar.RotatingMarker()),
                   ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
        progress = progressbar.ProgressBar(widgets=widgets, maxval=num_addrs)
        if show_progress:
            progress.start()
        self._cache = dict()
        seen = dict()
        for i, a in enumerate(self._addresses_to_check()):
            if show_progress:
                progress.update(i)
            try:
                bl = self.project.factory.block(a)
                if bl.size > self._max_block_size:
                    continue
                block_data = bl.bytes
            except (SimEngineError, SimMemoryError):
                continue
            if block_data in seen:
                self._cache[seen[block_data]].add(a)
                continue
            else:
                if len(bl.vex.constant_jump_targets) == 0 and not self._block_has_ip_relative(a, bl):
                    seen[block_data] = a
                    self._cache[a] = set()
                yield a
        if show_progress:
            progress.finish()

def upload(self, data):
        """
        ???????? ?????? ? ????????? (???????? ?? send)
        """
        if options.progress:
            written = 0
            start   = int(time.time())
            bar     = None

            try:
                total = int(self._content_length)
                if pclProgressBar:
                    try:
                        widgets = ["--> Upload: ", pclProgressBar.Percentage(), " ", pclProgressBar.Bar(left = "[", marker = "=", right = "]"), " ", pclProgressBar.ETA(), " ", pclProgressBar.FileTransferSpeed()]
                        bar = pclProgressBar.ProgressBar(widgets = widgets, maxval = total).start()
                    except:
                        total = pcl_human(total)
                else:
                    total = pcl_human(total)
            except:
                total = "-"

        datablock = data.read(self._options.chunk)

        while datablock:
            self.sock.sendall(datablock)

            if self._options.progress:
                written += len(datablock)
                if bar:
                    bar.update(written)
                else:
                    delta = int(time.time()) - start
                    if delta > 0:
                        sys.stderr.write("--> Upload: {0}/{1} ({2}/s){3}\r".format(pcl_human(written), total, pcl_human(written / delta), " " * 12))

            datablock = data.read(self._options.chunk)

        if self._options.progress:
            if bar:
                bar.finish()
            else:
                sys.stderr.write("{0}\r".format(" " * 33))

def pcl_query_download(options, response, filename):
    """
    ???????? ????? ?? ?????????

    ?????????:
        options  (pclOptions)   -- ????? ??????????
        response (HTTPResponse) -- HTTP ?????
        filename (str)          -- ??? ?????????? ????? ??? ??????
    """
    if options.progress:
        read  = 0
        start = int(time.time())
        bar   = None

        try:
            total = int(response.info().get("Content-Length"))
            if pclProgressBar:
                try:
                    widgets = ["--> Download: ", pclProgressBar.Percentage(), " ", pclProgressBar.Bar(left = "[", marker = "=", right = "]"), " ", pclProgressBar.ETA(), " ", pclProgressBar.FileTransferSpeed()]
                    bar = pclProgressBar.ProgressBar(widgets = widgets, maxval = total).start()
                except:
                    total = pcl_human(total)
            else:
                total = pcl_human(total)
        except:
            total = "-"

    with open(filename, "wb") as fd:
        while True:
            part = response.read(options.chunk)
            if not part:
                break

            fd.write(part)

            if options.progress:
                read += len(part)
                if bar:
                    bar.update(read)
                else:
                    delta = int(time.time()) - start
                    if delta > 0:
                        sys.stderr.write("--> Download: {0}/{1} ({2}/s){3}\r".format(pcl_human(read), total, pcl_human(read / delta), " " * 12))

    if options.progress:
        if bar:
            bar.finish()
        else:
            sys.stderr.write("{0}\r".format(" " * 35))

def run_collapse(self, hosts, cmd):
        progress = None
        if self.progressbar:
            from progressbar import ProgressBar, Percentage, Bar, ETA, FileTransferSpeed
            progress = ProgressBar(
                widgets=["Running: ", Percentage(), ' ', Bar(marker='.'), ' ', ETA(), ' ', FileTransferSpeed()],
                maxval=len(hosts))

        codes = {"total": 0, "error": 0, "success": 0}
        outputs = defaultdict(list)

        def worker(host, cmd):
            p = Popen(self.get_parallel_ssh_options(host, cmd), stdout=PIPE, stderr=PIPE)
            o = ""
            while True:
                outs, _, _ = select([p.stdout, p.stderr], [], [])
                outline = errline = ""
                if p.stdout in outs:
                    outline = p.stdout.readline()
                if p.stderr in outs:
                    errline = p.stderr.readline()
                o += outline + errline

                if outline == "" and errline == "" and p.poll() is not None:
                    break

            if o == "":
                o = colored("[ No Output ]\n", "yellow")
            outputs[o].append(host)
            if p.poll() == 0:
                codes["success"] += 1
            else:
                codes["error"] += 1
            codes["total"] += 1
            if self.progressbar:
                progress.update(codes["total"])

        pool = Pool(self.ssh_threads)
        if self.progressbar:
            progress.start()
        for host in hosts:
            pool.start(Greenlet(worker, host, cmd))

        try:
            pool.join()
        except KeyboardInterrupt:
            pass

        if self.progressbar:
            progress.finish()
        self.print_exec_results(codes)
        print()
        for output, hosts in outputs.items():
            msg = " %s    " % ','.join(hosts)
            table_width = min([len(msg) + 2, terminal_size()[0]])
            cprint("=" * table_width, "blue", attrs=["bold"])
            cprint(msg, "blue", attrs=["bold"])
            cprint("=" * table_width, "blue", attrs=["bold"])
            print(output)