我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tqdm.trange()。
def _generate_datasets(self): datasets = {'train': [], 'test': []} for dataset in self.datasets: print 'Generate dataset from {0}.'.format(dataset.name) # load dataset. dataset.load() # generate dataset for index in tqdm.trange(len(dataset), ascii=True): # get i-th data in the dataset. label, joint, image_file, image = dataset.get_data(index) # pad and crop image image, joint = self._pad_image(image, joint) image, joint = self._crop_image(image, joint) if not self._validate(joint): continue # save the image image_path = self._save_image(dataset.name, image_file, image) # write database line = self._make_dataset_line(image_path, joint) datasets[label].append(line) return datasets
def time_hdf5(): data_path = create_hdf5(BATCH_SIZE * NSTEPS) f = h5py.File(data_path) durs = [] for step in tqdm.trange(NSTEPS, desc='running hdf5'): start_time = time.time() arr = f['data'][BATCH_SIZE * step: BATCH_SIZE * (step+1)] read_time = time.time() arr = copy.deepcopy(arr) copy_time = time.time() durs.append(['hdf5 read', step, read_time - start_time]) durs.append(['hdf5 copy', step, copy_time - read_time]) f.close() os.remove(data_path) durs = pandas.DataFrame(durs, columns=['kind', 'stepno', 'dur']) return durs
def render(self, steps = 1, verbose = False): if len(self.cameras) == 0: self.cameras.append(([], {})) images = [[] for _ in self.cameras] depths = [[] for _ in self.cameras] masks = [[] for _ in self.cameras] for _ in trange(steps) if verbose else range(steps): for i, (args, kwargs) in enumerate(self.cameras): _, _, image, depth, mask = p.getCameraImage(self.width, self.height, *args, **kwargs) images[i].append(image) depths[i].append(depth) masks[i].append(mask) p.stepSimulation() if len(self.cameras) == 1: return images[0], depths[0], masks[0] else: return images, depths, masks
def fit(self, dataset, target=None): """ Fit loop. Args: target (np.ndarray, None): initial target of FQI. Returns: Last target computed. """ if self._boosted: if self._target is None: self._prediction = 0. self._next_q = 0. self._idx = 0 fit = self._fit_boosted else: fit = self._fit for _ in trange(self._n_iterations, dynamic_ncols=True, disable=self._quiet, leave=False): fit(dataset)
def train(self): tf.logging.info("Training starts...") self.data_loader.run_input_queue(self.sess) summary_writer = None for k in trange(self.max_step, desc="train"): fetch = { 'optim': self.model.optim, } result = self.model.train(self.sess, fetch, summary_writer) if result['step'] % self.log_step == 0: self._test(self.summary_writer) summary_writer = self._get_summary_writer(result) self.data_loader.stop_input_queue()
def train(self): z_fixed = np.random.normal(size=[self.batch_size*10, self.z_dim]) # samples of 10 times batch size gen = inf_train_gen(self.lines, self.batch_size, self.charmap) for step in trange(self.max_step): # Train generator _data = gen.next() summary_str, _ = self.sess.run([self.summary_op, self.g_optim], feed_dict={self.real_data: _data}) self.summary_writer.add_summary(summary_str, global_step=step) self.summary_writer.flush() # Train critic for i in range(self.critic_iters): _data = gen.next() self.sess.run(self.d_optim, feed_dict={self.real_data: _data}) if step % 100 == 99: _data = gen.next() g_loss, d_loss, slope = self.sess.run([self.g_loss, self.d_loss, self.slope], feed_dict={self.real_data: _data}) print("[{}/{}] Loss_D: {:.6f} Loss_G: {:.6f} Slope: {:.6f}". format(step+1, self.max_step, d_loss, g_loss, slope)) self.generate_samples(z_fixed, idx=step+1)
def load_to_ram(self, is_training): len_keys = self.len_train_keys if is_training else self.len_val_keys labs = np.empty([len_keys, 4], dtype=np.int32) poses = np.empty([len_keys,self.pshape[0],self.pshape[1],self.max_plen], dtype=np.float32) random_crop_bkp = self.random_crop random_pick_bkp = self.random_pick self.random_crop = False self.random_pick = False splitname = 'train' if is_training else 'val' print('Loading "%s" data to ram...' % splitname) t = trange(len_keys, dynamic_ncols=True) for k in t: key_idx, subject, action, pose, plen = self.read_h5_data(k, is_training) pose = pose[:, :, :self.max_plen] if plen > self.max_plen else pose plen = self.max_plen if plen > self.max_plen else plen labs[k, :] = [key_idx, subject, action, plen] poses[k, :, :, :plen] = pose self.random_crop = random_crop_bkp self.random_pick = random_pick_bkp return labs, poses
def compute_moments(pose_seq_input): labs, poses = pose_seq_input.load_to_ram(True) mask = np.zeros(np.shape(poses), dtype=np.bool) print('Computing Masks...') t = trange(len(pose_seq_input.train_keys), dynamic_ncols=True) for k in t: mask[k, :, :, :labs[k, 3]] = True poses = np.transpose(poses, [1, 2, 3, 0]) mask = np.transpose(mask, [1, 2, 3, 0]) masked_pose = np.reshape(poses[mask], [pose_seq_input.pshape[0], pose_seq_input.pshape[1], -1]) del mask del labs del poses print('Computing Moments...') skel_mean = np.reshape(np.mean(masked_pose, axis=(0, 2)), [1, 1, 1, 3]) skel_std = np.reshape(np.std(masked_pose, axis=(0, 2)), [1, 1, 1, 3]) del masked_pose print(skel_mean, skel_std) save_arr(pose_seq_input, skel_mean, 'skel_mean') save_arr(pose_seq_input, skel_std, 'skel_std')
def train(self, lr, iters, batch_size = 256): optimizer = optim.Adam(self.parameters(), lr=lr) t = trange(iters) for i in t: optimizer.zero_grad() inds = torch.floor(torch.rand(batch_size) * self.M).long().cuda() # bug: floor(rand()) sometimes gives 1 inds[inds >= self.M] = self.M - 1 inds = Variable(inds) loss = self.forward(inds) # print loss.data[0] t.set_description( str(loss.data[0]) ) loss.backward() optimizer.step() return self.state_model, self.goal_model
def train(self, lr, iters): optimizer = optim.Adam(self.parameters(), lr=lr) t = trange(iters) for i in t: optimizer.zero_grad() loss = self.forward( () ) # print loss.data[0] t.set_description( '%.3f | %.3f | %.3f | %.3f' % (self.mse, self.divergence, self.world_mse, self.location_mse) ) loss.backward() optimizer.step() U, V = self.__lookup() recon = torch.mm(U, V.t()) # print U, V, recon U = U.data.cpu().numpy() V = V.data.cpu().numpy() recon = recon.data.cpu().numpy() return U, V, recon
def train_loop(self,num_iter=None): ''' This is a function for handling the training of either CausalBEGAN or CausalGAN models. The python function Model.train_step() is called num_iter times and some general image save features: intervening, conditioning, etc are done here too. ''' num_iter=num_iter or self.model_config.num_iter #Train loop print('Entering train loop..') for counter in trange(num_iter): self.model.train_step(self.sess,counter) #scalar and histogram summaries if counter % self.config.log_step == 0: step,summ=self.sess.run([self.model.step,self.model.summary_op]) self.summary_writer.add_summary(summ,step) self.summary_writer.flush() #expensive summaries if counter % (self.config.log_step * 50) == 0: self.causal_sampling([8,16]) self.label_interpolation() self.sample_diversity() #more rare events if counter % (self.config.log_step * 100) == 0: self.causal_sampling([2,10]) ##Wrapper methods
def create_hdf5(n_img, path=None, shape=(IMSIZE, IMSIZE, 3), im_dtype=np.float32): if path is None: path = '/tmp' # os.makedirs(path) tempf = tempfile.NamedTemporaryFile(suffix='.hdf5', dir=path, delete=False) tempf.close() with h5py.File(tempf.name, 'w') as f: f.create_dataset('data', ((n_img, ) + shape), dtype=im_dtype) f.create_dataset('labels', (n_img, ), dtype=np.int64) img = np.random.randn(*shape) label = np.random.randint(1000) for i in tqdm.trange(n_img, desc='creating hdf5 file'): f['data'][i] = img f['labels'][i] = label return tempf.name
def time_tf(data): m = model.alexnet_nonorm(data.batch['data']) targets = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(m.output, data.batch['labels'])) init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) # start our custom queue runner's threads if hasattr(data, 'start_threads'): data.start_threads(sess) durs = [] for step in tqdm.trange(NSTEPS, desc='running ' + data.kind): start_time = time.time() if hasattr(data, 'start_threads') or not hasattr(data, 'next'): sess.run(targets) else: batch = data.next() feed_dict = {node: batch[name] for name, node in data.batch.items()} sess.run(targets, feed_dict=feed_dict) end_time = time.time() durs.append([data.kind, step, end_time - start_time]) if hasattr(data, 'stop_threads'): data.stop_threads(sess) sess.close() durs = pandas.DataFrame(durs, columns=['kind', 'stepno', 'dur']) return durs
def run_targets(sess, dbinterface, target_name, target, valid_loop, num_steps, online_agg_func, agg_func, save_intermediate_freq=None, validation_only=False): """TODO: this code resembles train() function, possible want to unify.""" agg_res = None if save_intermediate_freq is not None: n0 = len(dbinterface.outrecs) for _step in tqdm.trange(num_steps, desc=target_name): if valid_loop is not None: res = valid_loop(sess, target) else: res = sess.run(target) assert hasattr(res, 'keys'), 'result must be a dictionary' if save_intermediate_freq is not None and _step % save_intermediate_freq == 0: dbinterface.save(valid_res={target_name: res}, step=_step, validation_only=validation_only) agg_res = online_agg_func(agg_res, res, _step) result = agg_func(agg_res) if save_intermediate_freq is not None: dbinterface.sync_with_host() n1 = len(dbinterface.outrecs) result['intermediate_steps'] = dbinterface.outrecs[n0: n1] return result
def csv_rel2abs_path_convertor(csv_filenames: str, delimiter: str=' ', encoding='utf8') -> None: """ Convert relative paths into absolute paths :param csv_filenames: filename of csv :param delimiter: character to delimit felds in csv :param encoding: encoding format of csv file :return: """ for filename in tqdm(csv_filenames): absolute_path, basename = os.path.split(os.path.abspath(filename)) relative_paths = list() labels = list() # Reading CSV with open(filename, 'r', encoding=encoding) as f: csvreader = csv.reader(f, delimiter=delimiter) for row in csvreader: relative_paths.append(row[0]) labels.append(row[1]) # Writing converted_paths CSV export_filename = os.path.join(absolute_path, '{}_abs{}'.format(*os.path.splitext(basename))) with open(export_filename, 'w', encoding=encoding) as f: csvwriter = csv.writer(f, delimiter=delimiter) for i in trange(0, len(relative_paths)): csvwriter.writerow([os.path.abspath(os.path.join(absolute_path, relative_paths[i])), labels[i]])
def make_forecast(model: Sequential, look_back_buffer: numpy.ndarray, timesteps: int=1, batch_size: int=1): forecast_predict = numpy.empty((0, 1), dtype=numpy.float32) for _ in trange(timesteps, desc='predicting data\t', mininterval=1.0): # make prediction with current lookback buffer cur_predict = model.predict(look_back_buffer, batch_size) # add prediction to result forecast_predict = numpy.concatenate([forecast_predict, cur_predict], axis=0) # add new axis to prediction to make it suitable as input cur_predict = numpy.reshape(cur_predict, (cur_predict.shape[1], cur_predict.shape[0], 1)) # remove oldest prediction from buffer look_back_buffer = numpy.delete(look_back_buffer, 0, axis=1) # concat buffer with newest prediction look_back_buffer = numpy.concatenate([look_back_buffer, cur_predict], axis=1) return forecast_predict
def train(self): """ Train the model on the training set. A checkpoint of the model is saved after each epoch and if the validation accuracy is improved upon, a separate ckpt is created for use on the test set. """ # switch to train mode for dropout self.model.train() # load the most recent checkpoint if self.resume: self.load_checkpoint(best=False) for epoch in trange(self.start_epoch, self.epochs): # decay learning rate if self.is_decay: self.anneal_learning_rate(epoch) # train for 1 epoch self.train_one_epoch(epoch) # evaluate on validation set valid_acc = self.validate(epoch) is_best = valid_acc > self.best_valid_acc self.best_valid_acc = max(valid_acc, self.best_valid_acc) self.save_checkpoint({ 'epoch': epoch + 1, 'state_dict': self.model.state_dict(), 'best_valid_acc': self.best_valid_acc}, is_best)
def save_imshow_grid(images, logs_dir, filename, shape): """ Plot images in a grid of a given shape. """ fig = plt.figure(1) grid = ImageGrid(fig, 111, nrows_ncols=shape, axes_pad=0.05) size = shape[0] * shape[1] for i in trange(size, desc="Saving images"): grid[i].axis('off') grid[i].imshow(images[i]) plt.savefig(os.path.join(logs_dir, filename))
def get_bbox(bbox_dir, imglist, train_or_test): ret = [] ret_label = [] def parse_bbox(fname): root = ET.parse(fname).getroot() size = root.find('size').getchildren() size = map(int, [size[0].text, size[1].text]) box = root.find('object').find('bndbox').getchildren() box = map(lambda x: float(x.text), box) label = root.find('object').find('name').text return np.asarray(list(box), dtype='float32'),label with timed_operation('Loading Bounding Boxes ...'): cnt = 0 import tqdm for k in tqdm.trange(len(imglist)): fname = imglist[k][0] fname = fname[:-4] + 'xml' fname = os.path.join(bbox_dir,train_or_test ,fname) try: box,label = parse_bbox(fname) ret.append(box) ret_label.append(label) cnt += 1 except KeyboardInterrupt: raise except: ret.append(None) ret_label.append(-1) logger.info("{}/{} images have bounding box.".format(cnt, len(imglist))) return ret, ret_label
def __init__(self, cache=None, **kwargs): super(GTZAN, self).__init__(**kwargs) if kwargs.get('conf') is not None: conf = kwargs['conf'] cache = conf.get('cache', None) data_set_path = osp.join(DEFAULT_IMAGEST_BASE, self.data_set) self.data_set_path = data_set_path self.cache = cache X, y = parse_anno_file(data_set_path) if cache == 'raw': import librosa from tqdm import trange X_new = np.zeros((len(X), 1, 661500, 1)) for i in trange(len(X)): x,_ = librosa.load(osp.join(DEFAULT_DATA_BASE, X[i])) x_len = min(661500, len(x)) X_new[i,:,:x_len,0] = x[:x_len] if cache is not None and cache != 'raw': X = self.load_cache_X(X, cache) if cache == 'mfcc': X_new = np.zeros((len(X), X[0].shape[0], 1280, 1)) for i, x in enumerate(X): x_len = min(x.shape[1], 1280) X_new[i,:,:x_len,0] = x[:,:x_len] X = X_new # layout_X if self.layout_x == 'rel_path': self.X = X else: self.X = self.init_layout_X(X) # layout_y self.y = self.init_layout_y(y)
def prepare_multithread(self): """Preperation for mutithread processing.""" self.reset = False # num_batch_left should always be -1 until the last batch block of the epoch self.num_batch_left = -1 self.num_child = 10 self.child_processes = [None] * self.num_child self.batch_cursor_read = 0 self.batch_cursor_fetched = 0 # TODO: add this to cfg file self.prefetch_size = 5 # in terms of batch # TODO: may not need readed_batch after validating everything self.read_batch_array_size = self.total_batch + self.prefetch_size * self.batch_size self.readed_batch = Array('i', self.read_batch_array_size) for i in range(self.read_batch_array_size): self.readed_batch[i] = 0 self.prefetched_images = np.zeros((self.batch_size * self.prefetch_size * self.num_child, self.image_size, self.image_size, 3)) self.prefetched_labels = np.zeros( (self.batch_size * self.prefetch_size * self.num_child)) self.queue_in = [] self.queue_out = [] for i in range(self.num_child): self.queue_in.append(Queue()) self.queue_out.append(Queue()) self.start_process(i) self.start_prefetch(i) # fetch the first one desc = 'receive the first half: ' + \ str(self.num_child * self.prefetch_size / 2) + ' batches' for i in trange(self.num_child / 2, desc=desc): # print "collecting", i self.collect_prefetch(i)
def plot(self, samples, title): """ Plot core process time of chainer and pytorch. """ batch_sizes = [2**m for m in range(self.max_batch_index + 1)] process_time = {'chainer': {'mean': [], 'std': []}, 'pytorch': {'mean': [], 'std': []}} for batch_size in tqdm(batch_sizes, desc='batch size'): for name, process in tqdm(self.process.items(), desc='testers'): # set batch size. process.set_batch_size(batch_size) compute_time = [] # get compute time. for index in trange(samples, desc='samples'): start = time.time() process.run(self.only_inference) compute_time.append(time.time() - start) # calculate mean and std. process_time[name]['mean'].append(np.mean(compute_time)) process_time[name]['std'].append(np.std(compute_time)) # plot core process time of each batch size. for name, p_t in process_time.items(): plt.errorbar(batch_sizes, p_t['mean'], yerr=p_t['std'], label=name) # plot settings. plt.title(title) plt.legend(loc='lower right') plt.xlabel('batch size') plt.ylabel('core process time [sec]') # save plot. if self.debug: plt.show() else: filename = '_'.join(title.split(' ')) + '.png' plt.savefig(os.path.join(self.output, filename))
def segment(locs, info, segmentation, kwargs={}, callback=None): Y = info[0]['Height'] X = info[0]['Width'] n_frames = info[0]['Frames'] n_seg = n_segments(info, segmentation) bounds = _np.linspace(0, n_frames-1, n_seg+1, dtype=_np.uint32) segments = _np.zeros((n_seg, Y, X)) if callback is not None: callback(0) for i in _trange(n_seg, desc='Generating segments', unit='segments'): segment_locs = locs[(locs.frame >= bounds[i]) & (locs.frame < bounds[i+1])] _, segments[i] = render(segment_locs, info, **kwargs) if callback is not None: callback(i+1) return bounds, segments
def _sequence_data(self, Column=None): data = self.row_data if Column is None: self.Column = [] else: self.Column = Column for i in trange(data.nrows, desc='seq rows in memory'): Rows = {} for cell in trange(data.ncols, desc='seq cols in memory', leave=False): Rows[cell] = data.row_values(i)[cell] self.Column.append(Rows) return self.Column
def matrix_data(self, Column=None): data = self.sequence_data header = self._header_matrix_mapping(column=data) coordinate = self.coordinate row_start_point = coordinate['row_start_point'] col_title_length = coordinate['col_title_length'] if Column is None: self.Column = [] else: self.Column = Column title, comp = '', '' for row in trange(len(data), desc='matrix rows in memory'): if row > row_start_point: if data[row][header[self.item]] == None or data[row][header[self.item]] == '': Rows, config = {}, '' for cell in trange(len(data[row]), desc='matrix cols in memory', leave=False): if cell < col_title_length: Rows[cell] = data[row][cell] if cell >= col_title_length and str(data[row][cell]).lower() != self.tip: config += data[coordinate['row_start_point']][cell] + ';' # title if cell == header[self.item]: Rows[cell] = title # component if cell == header[self.component]: if data[row][header[self.component]] == None or data[row][header[self.component]] == '': Rows[cell] = comp else: comp = data[row][header[self.component]] Rows[col_title_length] = config self.Column.append(Rows) else: title = data[row][header[self.item]] return {'title': header, 'data': self.Column}
def config_data(self, Column=None, configColumn=None): data = self.sequence_data coordinate = self.coordinate sku_start_point = coordinate['sku_start_point'] #config ???? sku_end_point = coordinate['sku_end_point'] #config ???? sku_col_start_point = coordinate['sku_col_start_point'] #config ???? if Column is None: self.Column = [] else: self.Column = Column if configColumn is None: self.configColumn = [] else: self.configColumn = configColumn for i in tqdm(range(len(data))): # config ??????? if i >= sku_start_point and i <= sku_end_point: rows = OrderedDict() for k, v in dict(data[i]).items(): if k >= sku_col_start_point: rows.setdefault(k, v) self.Column.append(rows) for i in trange(sku_col_start_point+1, sku_col_start_point+len(self.Column[0]), desc='config rows in memory'): rows = OrderedDict() for row in trange(len(self.Column), desc='config cols in memory', leave=False): for k, v in dict(self.Column[row]).items(): if k == i: rows.setdefault(row, str(v).upper()) self.configColumn.append(rows) return {'column': self.Column, 'configcolumn': self.configColumn}
def create_copy_task_files(self, context_filename, answer_filename, vocab_size, num_examples, max_sequence_length): with open(context_filename, 'w') as file: for _ in trange(num_examples): num_tokens = np.random.randint(2, max_sequence_length, 1) tokens = np.random.randint(0, vocab_size, num_tokens) file.write(" ".join([str(x) for x in list(tokens)]) + "\n") shutil.copyfile(context_filename, answer_filename)
def summary_stock_tick_data(root_path, df, symbol, date_list): file_path = root_path + "/Data/CSV/tick/" + symbol + "/" out_file = root_path + "/Data/CSV/cashflow/" + symbol + ".csv" #pbar = trange(len(date_list), mininterval=0.1, smoothing=1, leave=False) #for i in pbar: for date in date_list: #date = date_list[i] start = time.time() file_name = file_path + symbol + "_" + date + ".csv" if os.path.exists(file_name) == False: continue try: data = pd.read_csv(file_name, index_col=0) except: print("error on symbol:", symbol, " date:", date) continue if (data is None) or data.empty or len(data) < 4: buy, sell, even = 0, 0, 0 else: buy_amount, sell_amount, even_amount, buy_volume, sell_volume, even_volume, buy_max, buy_min, buy_average, sell_max, sell_min, sell_average, even_max, even_min, even_average = group_tick_data_to_cashflow(data) df.loc[len(df)] = [date, symbol, buy_amount, sell_amount, even_amount, buy_volume, sell_volume, even_volume, buy_max, buy_min, buy_average, sell_max, sell_min, sell_average, even_max, even_min, even_average] #outMessage = '%s processed in: %.3s seconds' % (date, (time.time() - start)) #pbar.set_description(outMessage) df = df.sort_values(['symbol','date'], ascending=[True, True]) df.to_csv(out_file)
def process_data(root_path, symbols, dates): negative_pect = {} stock_memory = {} symbol_memory = {} range_len = 3 my_range = range(-1, -200, -1) #pbar = tqdm(total=len(my_range)) pbar = trange(len(my_range)) out_path = root_path + "/Data/CSV/target/" if os.path.exists(out_path) == False: os.mkdir(out_path) for index in my_range: day_range = [ dates[idx] for idx in range(index-range_len, index+1) ] file_name = out_path + day_range[-1] + ".csv" if os.path.exists(file_name): stock_filter = pd.read_csv(file_name, index_col=0) else: db_cashflow = process_all_stocks_data(root_path, symbols, day_range, stock_memory, symbol_memory, index, range_len) stock_filter = filter_cashflow(db_cashflow) if len(stock_filter) > 0: stock_filter.to_csv(file_name) negative_pect[day_range[-1]] = get_result(stock_filter) # outMessage = '%-*s processed in: %.4s seconds' % (6, index, (time.time() - startTime)) # pbar.set_description(outMessage) pbar.update(1) pbar.close() print(negative_pect)
def save_imshow_grid(images, logs_dir, filename, shape): """ Plot images in a grid of a given shape. """ pickle.dump(images, open(os.path.join(logs_dir, "image.pk"), "wb")) fig = plt.figure(1) grid = ImageGrid(fig, 111, nrows_ncols=shape, axes_pad=0.05) size = shape[0] * shape[1] for i in trange(size, desc="Saving images"): grid[i].axis('off') grid[i].imshow(images[i]) Image.fromarray(images[i]).save(os.path.join(logs_dir,str(i)),"jpeg") plt.savefig(os.path.join(logs_dir, filename))
def uuidpool(num, tablename): # generate 'num' uuid's, return array pool=[] for i in trange(num, desc=tablename): pool.append(str(uuid.uuid4())) return pool #------------------------------------------------------------------------------
def uuidpool(num, tablename): # generate 'num' uuid's, return array pool=[] for i in trange(num, desc=tablename): pool.append(str(uuid.uuid4())) return pool
def p_table (stations, datapoints): # Populate Table with db_r.Table('weatherstation_data').batch_writer() as batch: for station in trange(stations, desc='Stations'): for datapoint in trange(datapoints, desc='Datapoints'): item = item_gen(station) batch.put_item(Item=item) #------------------------------------------------------------------------------
def train(self): """ Run training function. Save model upon completion """ self.print_log('Training for %d epochs' % self.flags['num_epochs']) tf_inputs = (self.x['TRAIN'], self.im_dims['TRAIN'], self.gt_boxes['TRAIN']) self.step += 1 for self.epoch in trange(1, self.flags['num_epochs']+1, desc='epochs'): train_order = randomize_training_order(len(self.names['TRAIN'])) for i in tqdm(train_order): feed_dict = create_feed_dict(flags['data_directory'], self.names['TRAIN'], tf_inputs, i) # Run a training iteration if self.step % (self.flags['display_step']) == 0: # Record training metrics every display_step interval summary = self._record_train_metrics(feed_dict) self._record_training_step(summary) else: summary = self._run_train_iter(feed_dict) self._record_training_step(summary) ## Epoch finished # Save model if self.epoch % cfg.CHECKPOINT_RATE == 0: self._save_model(section=self.epoch) # Perform validation if self.epoch % cfg.VALID_RATE == 0: self.evaluate(test=False) # # Adjust learning rate # if self.epoch % cfg.TRAIN.LEARNING_RATE_DECAY_RATE == 0: # self.lr = self.lr * cfg.TRAIN.LEARNING_RATE_DECAY # self.print_log("Learning Rate: %f" % self.lr)
def process_digits(all_data, all_labels, data_directory, args): """ Generate data and saves in the appropriate format """ for s in range(len(flags['all_names'])): split = flags['all_names'][s] print('Processing {0} Data'.format(split)) key = 'train' if split == 'train' else 'eval' # Create writer (tf_records) or Image/Annotations/Names directories (PNGs) if args[key] == 'tfrecords': tf_writer = tf.python_io.TFRecordWriter(data_directory + 'clutteredMNIST_' + split + '.tfrecords') elif args[key] == 'PNG': make_Im_An_Na_directories(data_directory) else: raise ValueError('{0} is not a valid data format option'.format(args[key])) # Generate data for i in trange(flags['nums'][split]): # Generate cluttered MNIST image im_dims = [im_dims_generator(), im_dims_generator()] num_digits = num_digits_generator() img, gt_boxes = gen_nCluttered(all_data[s], all_labels[s], im_dims, num_digits) # Save data if args[key] == 'tfrecords': img = np.float32(img.flatten()).tostring() gt_boxes = np.int32(np.array(gt_boxes).flatten()).tostring() tf_write(img, gt_boxes, [flags['im_dims'], flags['im_dims']], tf_writer) elif args[key] == 'PNG': fname = split + '_img' + str(i) imsave(data_directory + 'Images/' + fname + '.png', np.float32(img)) np.savetxt(data_directory + 'Annotations/' + fname + '.txt', np.array(gt_boxes), fmt='%i') with open(data_directory + 'Names/' + split + '.txt', 'a') as f: f.write(fname + '\n') ############################################################################### # Image generation functions ###############################################################################
def sample(L, p, samples=1000, cutoff=200): '''Repeated single shot corrections for the toric code with perfect measurements. Return an array of nb of cycles until failure for a given L and p.''' results = [] for _ in trange(samples, desc='%d; %.2f'%(L,p), leave=False): code = ToricCode(L) i = 1 while code.step_error_and_perfect_correction(p) and i<cutoff: i+=1 results.append(i) return np.array(results, dtype=int)
def run_benchmark(input_data): print("Running dry runs...") for n in trange(args.dry_runs): iteration(input_data) print("\n Running measured runs...") running_time = 0 for n in trange(args.runs): start, end = iteration(input_data) running_time += end - start return running_time / float(args.runs)
def _maybe_generate_and_save(self, except_list=[]): self.data = {} for name, num in self.data_num.items(): if name in except_list: tf.logging.info("Skip creating {} because of given except_list {}".format(name, except_list)) continue path = self.get_path(name) if not os.path.exists(path): tf.logging.info("Creating {} for [{}]".format(path, self.task)) x = np.zeros([num, self.max_length, 2], dtype=np.float32) y = np.zeros([num, self.max_length], dtype=np.int32) for idx in trange(num, desc="Create {} data".format(name)): n_nodes = self.rng.randint(self.min_length, self.max_length+ 1) nodes, res = generate_one_example(n_nodes, self.rng) x[idx,:len(nodes)] = nodes y[idx,:len(res)] = res np.savez(path, x=x, y=y) self.data[name] = TSP(x=x, y=y, name=name) else: tf.logging.info("Skip creating {} for [{}]".format(path, self.task)) tmp = np.load(path) self.data[name] = TSP(x=tmp['x'], y=tmp['y'], name=name)
def test_memory(insertions, samples, img_shape, misc_len, batch_size, capacity, img_dtype=np.float32): print("image shape:", img_shape) print("misc vector lenght:", misc_len) print("batchsize:", batch_size) print("capacity:", capacity) print("image data type:", img_dtype.__name__) memory = ReplayMemory(img_shape, misc_len, capacity, batch_size) if img_dtype != np.float32: s = [(np.random.random(img_shape) * 255).astype(img_dtype), np.random.random(misc_len).astype(np.float32)] s2 = [(np.random.random(img_shape) * 255).astype(img_dtype), np.random.random(misc_len).astype(np.float32)] else: s = [np.random.random(img_shape).astype(img_dtype), np.random.random(misc_len).astype(np.float32)] s2 = [np.random.random(img_shape).astype(img_dtype), np.random.random(misc_len).astype(np.float32)] a = 0 r = 1.0 terminal = False for _ in trange(capacity, leave=False, desc="Prefilling memory."): memory.add_transition(s, a, s2, r, terminal) start = time() for _ in trange(insertions, leave=False, desc="Testing insertions speed"): memory.add_transition(s, a, s2, r, terminal) inserts_time = time() - start start = time() for _ in trange(samples, leave=False, desc="Testing sampling speed"): sample = memory.get_sample() sample_time = time() - start print("\t{:0.1f} insertions/s. 1k insertions in: {:0.2f}s".format(insertions / inserts_time, inserts_time / insertions * 1000)) print("\t{:0.1f} samples/s. 1k samples in: {:0.2f}s".format(samples / sample_time, sample_time / samples * 1000)) print()
def test(self, episodes_num=None, deterministic=True): if episodes_num is None: episodes_num = self.test_episodes_per_epoch test_start_time = time.time() test_rewards = [] test_actions = [] test_frameskips = [] for _ in trange(episodes_num, desc="Testing", file=sys.stdout, leave=False, disable=not self.enable_progress_bar): total_reward, actions, frameskips, _ = self.run_episode(deterministic=deterministic, return_stats=True) test_rewards.append(total_reward) test_actions += actions test_frameskips += frameskips self.doom_wrapper.reset() if self.local_network.has_state(): self.local_network.reset_state() test_end_time = time.time() test_duration = test_end_time - test_start_time min_score = np.min(test_rewards) max_score = np.max(test_rewards) mean_score = np.mean(test_rewards) score_std = np.std(test_rewards) log( "TEST: mean: {}, min: {}, max: {}, test time: {}".format( green("{:0.3f}±{:0.2f}".format(mean_score, score_std)), red("{:0.3f}".format(min_score)), blue("{:0.3f}".format(max_score)), sec_to_str(test_duration))) return test_rewards, test_actions, test_frameskips
def train(self): max_epoch = int(math.ceil(1. * self.max_iter / len(self.train_loader))) for epoch in tqdm.trange(self.epoch, max_epoch, desc='Train', ncols=80): self.epoch = epoch self.train_epoch() if self.iteration >= self.max_iter: break
def export_to_tf(self): def make_example(key_idx, subject, action, pose, plen): ex = tf.train.Example() ex.features.feature["key_idx"].int64_list.value.append(int(key_idx)) ex.features.feature["subject"].int64_list.value.append(int(subject)) ex.features.feature["action"].int64_list.value.append(int(action)) ex.features.feature["plen"].int64_list.value.append(int(plen)) for sublist in pose.tolist(): for subsublist in sublist: for value in subsublist: ex.features.feature["pose"].float_list.value.append(value) return ex def write_split(is_training, keys): writer = None shard = 0 splitname = 'train' if is_training else 'val' print('Transforming "%s" split...' % splitname) t = trange(len(keys), dynamic_ncols=True) for k in t: if writer == None: writer = tf.python_io.TFRecordWriter( os.path.join(self.data_path, self.data_set + '_' + splitname + '_shard' + str(shard) + '.tf') ) key_idx, subject, action, pose, plen = self.read_h5_data(k, is_training) ex = make_example(key_idx, subject, action, pose, plen) writer.write(ex.SerializeToString()) if ((k + 1) % 4096) == 0: writer.close() writer = None shard += 1 if writer != None: writer.close() write_split(True, self.train_keys) write_split(False, self.val_keys)
def main_loop(self): # some final operations that might modify the graph self._init_summary() get_global_step_var() # ensure there is such var, before finalizing the graph logger.info("Setup callbacks ...") callbacks = self.config.callbacks callbacks.setup_graph(self) # TODO use weakref instead? logger.info("Initializing graph variables ...") self.sess.run(tf.initialize_all_variables()) self.config.session_init.init(self.sess) tf.get_default_graph().finalize() self._start_concurrency() with self.sess.as_default(): try: self.global_step = get_global_step() logger.info("Start training with global_step={}".format(self.global_step)) callbacks.before_train() for epoch in range(self.config.starting_epoch, self.config.max_epoch+1): with timed_operation( 'Epoch {}, global_step={}'.format( epoch, self.global_step + self.config.step_per_epoch)): for step in tqdm.trange( self.config.step_per_epoch, **get_tqdm_kwargs(leave=True)): if self.coord.should_stop(): return self.run_step() #callbacks.trigger_step() # not useful? self.global_step += 1 self.trigger_epoch() except (KeyboardInterrupt, Exception): raise finally: # Do I need to run queue.close? callbacks.after_train() self.coord.request_stop() self.summary_writer.close() self.sess.close()
def get_training_bbox(bbox_dir, imglist): ret = [] def parse_bbox(fname): root = ET.parse(fname).getroot() size = root.find('size').getchildren() size = map(int, [size[0].text, size[1].text]) box = root.find('object').find('bndbox').getchildren() box = map(lambda x: float(x.text), box) #box[0] /= size[0] #box[1] /= size[1] #box[2] /= size[0] #box[3] /= size[1] return np.asarray(box, dtype='float32') with timed_operation('Loading Bounding Boxes ...'): cnt = 0 import tqdm for k in tqdm.trange(len(imglist)): fname = imglist[k][0] fname = fname[:-4] + 'xml' fname = os.path.join(bbox_dir, fname) try: ret.append(parse_bbox(fname)) cnt += 1 except KeyboardInterrupt: raise except: ret.append(None) logger.info("{}/{} images have bounding box.".format(cnt, len(imglist))) return ret
