我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.python.ops.control_flow_ops.with_dependencies()。
def _crop(image, offset_height, offset_width, crop_height, crop_width): original_shape = tf.shape(image) rank_assertion = tf.Assert( tf.equal(tf.rank(image), 3), ['Rank of image must be equal to 3.']) cropped_shape = control_flow_ops.with_dependencies( [rank_assertion], tf.stack([crop_height, crop_width, original_shape[2]])) size_assertion = tf.Assert( tf.logical_and( tf.greater_equal(original_shape[0], crop_height), tf.greater_equal(original_shape[1], crop_width)), ['Crop size greater than the image size.']) offsets = tf.to_int32(tf.stack([offset_height, offset_width, 0])) # Use tf.slice instead of crop_to_bounding box as it accepts tensors to # define the crop size. image = control_flow_ops.with_dependencies([size_assertion], tf.slice(image, offsets, cropped_shape)) return tf.reshape(image, cropped_shape)
def _crop(image, offset_height, offset_width, crop_height, crop_width): original_shape = tf.shape(image) rank_assertion = tf.Assert( tf.equal(tf.rank(image), 3), ['Rank of image must be equal to 3.']) cropped_shape = control_flow_ops.with_dependencies( [rank_assertion], tf.stack([crop_height, crop_width, original_shape[2]])) size_assertion = tf.Assert( tf.logical_and( tf.greater_equal(original_shape[0], crop_height), tf.greater_equal(original_shape[1], crop_width)), ['Crop size greater than the image size.']) offsets = tf.to_int32(tf.stack([offset_height, offset_width, 0])) # Use tf.slice instead of crop_to_bounding box as it accepts tensors to # define the crop size. image = control_flow_ops.with_dependencies( [size_assertion], tf.slice(image, offsets, cropped_shape)) return tf.reshape(image, cropped_shape)
def _assert_non_negative_int32_scalar(self, x): """Helper which ensures that input is a non-negative, int32, scalar.""" x = ops.convert_to_tensor(x, name="x") if x.dtype.base_dtype != dtypes.int32.base_dtype: raise TypeError("%s.dtype=%s is not %s" % (x.name, x.dtype, dtypes.int32)) x_value_static = tensor_util.constant_value(x) if x.get_shape().ndims is not None and x_value_static is not None: if x.get_shape().ndims != 0: raise ValueError("%s.ndims=%d is not 0 (scalar)" % (x.name, x.get_shape().ndims)) if x_value_static < 0: raise ValueError("%s.value=%d cannot be negative" % (x.name, x_value_static)) return x if self.validate_args: x = control_flow_ops.with_dependencies([ check_ops.assert_rank(x, 0), check_ops.assert_non_negative(x)], x) return x
def _check_shape(self, shape): """Check that the init arg `shape` defines a valid operator.""" shape = ops.convert_to_tensor(shape, name="shape") if not self._verify_pd: return shape # Further checks are equivalent to verification that this is positive # definite. Why? Because the further checks simply check that this is a # square matrix, and combining the fact that this is square (and thus maps # a vector space R^k onto itself), with the behavior of .matmul(), this must # be the identity operator. rank = array_ops.size(shape) assert_matrix = check_ops.assert_less_equal(2, rank) with ops.control_dependencies([assert_matrix]): last_dim = array_ops.gather(shape, rank - 1) second_to_last_dim = array_ops.gather(shape, rank - 2) assert_square = check_ops.assert_equal(last_dim, second_to_last_dim) return control_flow_ops.with_dependencies([assert_matrix, assert_square], shape)
def _variance(self): var = (self._ones() * math_ops.square(self.sigma) * self.df / (self.df - 2)) # When 1 < df <= 2, variance is infinite. inf = np.array(np.inf, dtype=self.dtype.as_numpy_dtype()) result_where_defined = math_ops.select( math_ops.greater(self.df, array_ops.fill(self.batch_shape(), 2.)), var, array_ops.fill(self.batch_shape(), inf, name="inf")) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) return math_ops.select( math_ops.greater(self.df, self._ones()), result_where_defined, array_ops.fill(self.batch_shape(), nan, name="nan")) else: return control_flow_ops.with_dependencies([ check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.df, message="variance not defined for components of df <= 1"), ], result_where_defined)
def _check_chol(self, chol): """Verify that `chol` is proper.""" chol = ops.convert_to_tensor(chol, name="chol") if not self.verify_pd: return chol shape = array_ops.shape(chol) rank = array_ops.rank(chol) is_matrix = check_ops.assert_rank_at_least(chol, 2) is_square = check_ops.assert_equal( array_ops.gather(shape, rank - 2), array_ops.gather(shape, rank - 1)) deps = [is_matrix, is_square] diag = array_ops.matrix_diag_part(chol) deps.append(check_ops.assert_positive(diag)) return control_flow_ops.with_dependencies(deps, chol)
def _mode(self): mode = ((self.alpha - 1.) / (array_ops.expand_dims(self.alpha_sum, dim=-1) - math_ops.cast(self.event_shape()[0], self.dtype))) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) shape = array_ops.concat(0, (self.batch_shape(), self.event_shape())) return math_ops.select( math_ops.greater(self.alpha, 1.), mode, array_ops.fill(shape, nan, name="nan")) else: return control_flow_ops.with_dependencies([ check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.alpha, message="mode not defined for components of alpha <= 1") ], mode)
def _get_train_ops(self, features, _): (_, _, losses, training_op) = clustering_ops.KMeans( self._parse_tensor_or_dict(features), self._num_clusters, self._training_initial_clusters, self._distance_metric, self._use_mini_batch, random_seed=self._random_seed, kmeans_plus_plus_num_retries=self.kmeans_plus_plus_num_retries ).training_graph() incr_step = tf.assign_add(tf.contrib.framework.get_global_step(), 1) self._loss = tf.reduce_sum(losses) tf.scalar_summary('loss/raw', self._loss) training_op = with_dependencies([training_op, incr_step], self._loss) return training_op, self._loss
def _mode(self): mode = (self.a - 1.)/ (self.a_b_sum - 2.) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) return math_ops.select( math_ops.logical_and( math_ops.greater(self.a, 1.), math_ops.greater(self.b, 1.)), mode, array_ops.fill(self.batch_shape(), nan, name="nan")) else: return control_flow_ops.with_dependencies([ check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.a, message="Mode not defined for components of a <= 1."), check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.b, message="Mode not defined for components of b <= 1."), ], mode)
def build_training_op(self, optimizer, _log): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) if update_ops: updates = tf.group(*update_ops) loss = control_flow_ops.with_dependencies([updates], self.loss) else: loss = self.loss with tf.name_scope('optimizer'): self.learning_rate = tf.placeholder(tf.float32, shape=[], name='learning_rate') if optimizer=='adam': optimizer = tf.train.AdamOptimizer(self.learning_rate) elif optimizer=='momentum': optimizer = tf.train.MomentumOptimizer(self.learning_rate, momentum=0.9, use_nesterov=True) else: raise ValueError("Invalid optimizer option") self.train_op = optimizer.minimize(loss, global_step=self.global_step)
def Print(tensor, data, msg='', file=None, mode='w'): from tensorflow.python.ops import control_flow_ops import util def np_print(*args): if util.str.contains(msg, '%'): message = msg % tuple(args) else: message = msg + ' %' * len(args) % tuple(args) if file is not None: file_path = util.io.get_absolute_path(file) print('writting message to file(%s):' % (file_path), message) with open(file_path, mode) as f: print(message, file=f) else: print(message) return control_flow_ops.with_dependencies([tf.py_func(np_print, data, [])], tensor)
def createUpdateOp(gradClip=1): with tf.name_scope("optimizer"): optimizer=tf.train.AdamOptimizer(learning_rate=opt.learningRate, epsilon=opt.adamEps) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) totalLoss = slim.losses.get_total_loss() grads = optimizer.compute_gradients(totalLoss, var_list=net.getVariables()) if gradClip is not None: cGrads = [] for g, v in grads: if g is None: print("WARNING: no grad for variable "+v.op.name) continue cGrads.append((tf.clip_by_value(g, -float(gradClip), float(gradClip)), v)) grads = cGrads update_ops.append(optimizer.apply_gradients(grads)) return control_flow_ops.with_dependencies([tf.group(*update_ops)], totalLoss, name='train_op')
def _model_builder(self): """Creates a model function.""" def _model_fn(features, labels, mode): """Model function.""" assert labels is None, labels (all_scores, model_predictions, losses, training_op) = gmm_ops.gmm( self._parse_tensor_or_dict(features), self._training_initial_clusters, self._num_clusters, self._random_seed, self._covariance_type, self._params) incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses) training_op = with_dependencies([training_op, incr_step], loss) predictions = { GMM.ALL_SCORES: all_scores[0], GMM.ASSIGNMENTS: model_predictions[0][0], } eval_metric_ops = { GMM.SCORES: _streaming_sum(loss), } return model_fn_lib.ModelFnOps(mode=mode, predictions=predictions, eval_metric_ops=eval_metric_ops, loss=loss, train_op=training_op) return _model_fn
def _mean(self): mean = self.loc * array_ops.ones(self.batch_shape(), dtype=self.dtype) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) return array_ops.where( math_ops.greater( self.df, array_ops.ones(self.batch_shape(), dtype=self.dtype)), mean, array_ops.fill(self.batch_shape(), nan, name="nan")) else: return control_flow_ops.with_dependencies( [ check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.df, message="mean not defined for components of df <= 1"), ], mean)
def _mode(self): mode = (self.a - 1.)/ (self.a_b_sum - 2.) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) return array_ops.where( math_ops.logical_and( math_ops.greater(self.a, 1.), math_ops.greater(self.b, 1.)), mode, array_ops.fill(self.batch_shape(), nan, name="nan")) else: return control_flow_ops.with_dependencies([ check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.a, message="Mode not defined for components of a <= 1."), check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.b, message="Mode not defined for components of b <= 1."), ], mode)
def _sample_n(self, n, seed=None): n_draws = math_ops.cast(self.total_count, dtype=dtypes.int32) if self.total_count.get_shape().ndims is not None: if self.total_count.get_shape().ndims != 0: raise NotImplementedError( "Sample only supported for scalar number of draws.") elif self.validate_args: is_scalar = check_ops.assert_rank( n_draws, 0, message="Sample only supported for scalar number of draws.") n_draws = control_flow_ops.with_dependencies([is_scalar], n_draws) k = self.event_shape()[0] # Flatten batch dims so logits has shape [B, k], # where B = reduce_prod(self.batch_shape()). draws = random_ops.multinomial( logits=array_ops.reshape(self.logits, [-1, k]), num_samples=n * n_draws, seed=seed) draws = array_ops.reshape(draws, shape=[-1, n, n_draws]) x = math_ops.reduce_sum(array_ops.one_hot(draws, depth=k), reduction_indices=-2) # shape: [B, n, k] x = array_ops.transpose(x, perm=[1, 0, 2]) final_shape = array_ops.concat([[n], self.batch_shape(), [k]], 0) return array_ops.reshape(x, final_shape)
def _mode(self): mode = ((self.alpha - 1.) / (array_ops.expand_dims(self.alpha_sum, dim=-1) - math_ops.cast(self.event_shape()[0], self.dtype))) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) shape = array_ops.concat((self.batch_shape(), self.event_shape()), 0) return array_ops.where( math_ops.greater(self.alpha, 1.), mode, array_ops.fill(shape, nan, name="nan")) else: return control_flow_ops.with_dependencies([ check_ops.assert_less( array_ops.ones((), dtype=self.dtype), self.alpha, message="mode not defined for components of alpha <= 1") ], mode)
def test_delayed_update_moving_vars_and_output(): with tf.Session() as sess: height, width = 3, 3 image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) images_mean = np.mean(image_values, axis=(0, 1, 2)) images_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) decay = 0.8 epsilon = 1e-5 output_s = batch_norm(images, is_training=True, reuse=None, decay=decay, epsilon=epsilon, updates_collections=tf.GraphKeys.UPDATE_OPS, name='BatchNorm') update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # updates_ops are added to UPDATE_OPS collection. assert len(update_ops) == 2 with tf.control_dependencies(update_ops): barrier = tf.no_op(name='barrier') output_s = control_flow_ops.with_dependencies([barrier], output_s) sess.run(tf.global_variables_initializer()) moving_mean = tf.contrib.framework.get_variables('BatchNorm/moving_mean')[0] moving_variance = tf.contrib.framework.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. assert_array_almost_equal(mean, [0] * 3) assert_array_almost_equal(variance, [1] * 3) # Feed in the same batch of images 10 times n_times = 10 expected_mean = np.array([0.] * 3) expected_var = np.array([1.] * 3) expected_output = (image_values - images_mean) / np.sqrt(images_var + epsilon) for _ in xrange(n_times): output = sess.run(output_s) mean, variance = sess.run([moving_mean, moving_variance]) expected_mean = expected_mean * decay + images_mean * (1 - decay) expected_var = expected_var * decay + images_var * (1 - decay) assert_array_almost_equal(output, expected_output) assert_array_almost_equal(mean, expected_mean) assert_array_almost_equal(variance, expected_var)
def test_delayed_update_moving_vars(): with tf.Session() as sess: height, width = 3, 3 image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) decay = 0.1 epsilon = 1e-5 output = batch_norm(images, is_training=True, reuse=None, decay=decay, epsilon=epsilon, updates_collections=tf.GraphKeys.UPDATE_OPS, name='BatchNorm') update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # updates_ops are added to UPDATE_OPS collection. assert len(update_ops) == 2 with tf.control_dependencies(update_ops): barrier = tf.no_op(name='barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.global_variables_initializer()) moving_mean = tf.contrib.framework.get_variables('BatchNorm/moving_mean')[0] moving_variance = tf.contrib.framework.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. assert_array_almost_equal(mean, [0] * 3) assert_array_almost_equal(variance, [1] * 3) for _ in range(10): sess.run([output]) mean = moving_mean.eval() variance = moving_variance.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_variance == expected_var. assert_array_almost_equal(mean, expected_mean) assert_array_almost_equal(variance, expected_var)
def test_delayed_update_moving_vars(): with tf.Session() as sess: epsilon = 1e-5 height, width = 3, 3 image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_inv_std = 1.0 / np.sqrt(np.var(image_values, axis=(0, 1, 2)) + epsilon) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) decay = 0.1 output = batch_norm(images, True, None, decay=decay, epsilon=epsilon, name="BatchNorm", updates_collections=tf.GraphKeys.UPDATE_OPS) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # updates_ops are added to UPDATE_OPS collection. assert len(update_ops) == 2 with tf.control_dependencies(update_ops): barrier = tf.no_op(name='barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.global_variables_initializer()) moving_mean = tf.contrib.framework.get_variables('BatchNorm/moving_mean')[0] moving_inv_std = tf.contrib.framework.get_variables('BatchNorm/moving_inv_std')[0] mean, inv_std = sess.run([moving_mean, moving_inv_std]) # After initialization moving_mean == 0 and moving_variance == 1. assert_array_almost_equal(mean, [0] * 3) assert_array_almost_equal(inv_std, [1] * 3) for _ in range(10): sess.run([output]) mean = moving_mean.eval() inv_std = moving_inv_std.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_inv_std == expected_inv_std. assert_array_almost_equal(mean, expected_mean) assert_array_almost_equal(inv_std, expected_inv_std)
def testComputeMovingVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output = ops.batch_norm(images, decay=0.1) update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION) with tf.control_dependencies(update_ops): barrier = tf.no_op(name='gradient_barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.initialize_all_variables()) moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 3) self.assertAllClose(variance, [1] * 3) for _ in range(10): sess.run([output]) mean = moving_mean.eval() variance = moving_variance.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_variance == expected_var. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def testEvalMovingVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output = ops.batch_norm(images, decay=0.1, is_training=False) update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION) with tf.control_dependencies(update_ops): barrier = tf.no_op(name='gradient_barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.initialize_all_variables()) moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 3) self.assertAllClose(variance, [1] * 3) # Simulate assigment from saver restore. init_assigns = [tf.assign(moving_mean, expected_mean), tf.assign(moving_variance, expected_var)] sess.run(init_assigns) for _ in range(10): sess.run([output], {images: np.random.rand(*image_shape)}) mean = moving_mean.eval() variance = moving_variance.eval() # Although we feed different images, the moving_mean and moving_variance # shouldn't change. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def testReuseVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output = ops.batch_norm(images, decay=0.1, is_training=False) update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION) with tf.control_dependencies(update_ops): barrier = tf.no_op(name='gradient_barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.initialize_all_variables()) moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 3) self.assertAllClose(variance, [1] * 3) # Simulate assigment from saver restore. init_assigns = [tf.assign(moving_mean, expected_mean), tf.assign(moving_variance, expected_var)] sess.run(init_assigns) for _ in range(10): sess.run([output], {images: np.random.rand(*image_shape)}) mean = moving_mean.eval() variance = moving_variance.eval() # Although we feed different images, the moving_mean and moving_variance # shouldn't change. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def testComputeMovingVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output = ops.batch_norm(images, decay=0.1) update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION) with tf.control_dependencies(update_ops): barrier = tf.no_op(name='gradient_barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.global_variables_initializer()) moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 3) self.assertAllClose(variance, [1] * 3) for _ in range(10): sess.run([output]) mean = moving_mean.eval() variance = moving_variance.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_variance == expected_var. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def testEvalMovingVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output = ops.batch_norm(images, decay=0.1, is_training=False) update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION) with tf.control_dependencies(update_ops): barrier = tf.no_op(name='gradient_barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.global_variables_initializer()) moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 3) self.assertAllClose(variance, [1] * 3) # Simulate assigment from saver restore. init_assigns = [tf.assign(moving_mean, expected_mean), tf.assign(moving_variance, expected_var)] sess.run(init_assigns) for _ in range(10): sess.run([output], {images: np.random.rand(*image_shape)}) mean = moving_mean.eval() variance = moving_variance.eval() # Although we feed different images, the moving_mean and moving_variance # shouldn't change. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def testReuseVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output = ops.batch_norm(images, decay=0.1, is_training=False) update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION) with tf.control_dependencies(update_ops): barrier = tf.no_op(name='gradient_barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.global_variables_initializer()) moving_mean = variables.get_variables('BatchNorm/moving_mean')[0] moving_variance = variables.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 3) self.assertAllClose(variance, [1] * 3) # Simulate assigment from saver restore. init_assigns = [tf.assign(moving_mean, expected_mean), tf.assign(moving_variance, expected_var)] sess.run(init_assigns) for _ in range(10): sess.run([output], {images: np.random.rand(*image_shape)}) mean = moving_mean.eval() variance = moving_variance.eval() # Although we feed different images, the moving_mean and moving_variance # shouldn't change. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def random_crop(value, size, seed=None, name=None): """Randomly crops a tensor to a given size. Slices a shape `size` portion out of `value` at a uniformly chosen offset. Requires `value.shape >= size`. If a dimension should not be cropped, pass the full size of that dimension. For example, RGB images can be cropped with `size = [crop_height, crop_width, 3]`. Args: value: Input tensor to crop. size: 1-D tensor with size the rank of `value`. seed: Python integer. Used to create a random seed. See @{tf.set_random_seed} for behavior. name: A name for this operation (optional). Returns: A cropped tensor of the same rank as `value` and shape `size`. """ # TODO(shlens): Implement edge case to guarantee output size dimensions. # If size > value.shape, zero pad the result so that it always has shape # exactly size. with ops.name_scope(name, "random_crop", [value, size]) as name: value = ops.convert_to_tensor(value, name="value") size = ops.convert_to_tensor(size, dtype=dtypes.int32, name="size") shape = array_ops.shape(value) check = control_flow_ops.Assert( math_ops.reduce_all(shape >= size), ["Need value.shape >= size, got ", shape, size], summarize=1000) shape = control_flow_ops.with_dependencies([check], shape) limit = shape - size + 1 offset = random_uniform( array_ops.shape(shape), dtype=size.dtype, maxval=size.dtype.max, seed=seed) % limit return array_ops.slice(value, offset, size, name=name)
def test_delayed_update_moving_vars_and_output(): with tf.Session() as sess: height, width = 3, 3 image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) images_mean = np.mean(image_values, axis=(0, 1, 2)) images_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) decay = 0.8 epsilon = 1e-5 output_s = batch_norm(images, is_training=True, reuse=None, decay=decay, epsilon=epsilon, updates_collections=tf.GraphKeys.UPDATE_OPS, name='BatchNorm') update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # updates_ops are added to UPDATE_OPS collection. assert len(update_ops) == 2 with tf.control_dependencies(update_ops): barrier = tf.no_op(name='barrier') output_s = control_flow_ops.with_dependencies([barrier], output_s) sess.run(tf.global_variables_initializer()) moving_mean = tf.contrib.framework.get_variables('BatchNorm/moving_mean')[0] moving_variance = tf.contrib.framework.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. assert_array_almost_equal(mean, [0] * 3) assert_array_almost_equal(variance, [1] * 3) # Feed in the same batch of images 10 times n_times = 10 expected_mean = np.array([0.] * 3) expected_var = np.array([1.] * 3) expected_output = (image_values - images_mean) / np.sqrt(images_var + epsilon) for _ in range(n_times): output = sess.run(output_s) mean, variance = sess.run([moving_mean, moving_variance]) expected_mean = expected_mean * decay + images_mean * (1 - decay) expected_var = expected_var * decay + images_var * (1 - decay) assert_array_almost_equal(output, expected_output, decimal=4) assert_array_almost_equal(mean, expected_mean, decimal=4) assert_array_almost_equal(variance, expected_var, decimal=4)
def test_delayed_update_moving_vars(): with tf.Session() as sess: height, width = 3, 3 image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) decay = 0.1 epsilon = 1e-5 output = batch_norm(images, is_training=True, reuse=None, decay=decay, epsilon=epsilon, updates_collections=tf.GraphKeys.UPDATE_OPS, name='BatchNorm') update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # updates_ops are added to UPDATE_OPS collection. assert len(update_ops) == 2 with tf.control_dependencies(update_ops): barrier = tf.no_op(name='barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.global_variables_initializer()) moving_mean = tf.contrib.framework.get_variables('BatchNorm/moving_mean')[0] moving_variance = tf.contrib.framework.get_variables('BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. assert_array_almost_equal(mean, [0] * 3) assert_array_almost_equal(variance, [1] * 3) for _ in range(10): sess.run([output]) mean = moving_mean.eval() variance = moving_variance.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_variance == expected_var. assert_array_almost_equal(mean, expected_mean, decimal=4) assert_array_almost_equal(variance, expected_var, decimal=4)
def test_delayed_update_moving_vars(): with tf.Session() as sess: epsilon = 1e-5 height, width = 3, 3 image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_inv_std = 1.0 / np.sqrt(np.var(image_values, axis=(0, 1, 2)) + epsilon) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) decay = 0.1 output = batch_norm(images, True, None, decay=decay, epsilon=epsilon, name="BatchNorm") update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # updates_ops are added to UPDATE_OPS collection. assert len(update_ops) == 2 with tf.control_dependencies(update_ops): barrier = tf.no_op(name='barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()]) moving_mean = tf.contrib.framework.get_variables('BatchNorm/moving_mean')[0] moving_inv_std = tf.contrib.framework.get_variables('BatchNorm/moving_inv_std')[0] mean, inv_std = sess.run([moving_mean, moving_inv_std]) # After initialization moving_mean == 0 and moving_variance == 1. assert_array_almost_equal(mean, [0] * 3) assert_array_almost_equal(inv_std, [1] * 3) for _ in range(10): sess.run([output]) mean = moving_mean.eval() inv_std = moving_inv_std.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_inv_std == expected_inv_std. assert_array_almost_equal(mean, expected_mean, decimal=4) assert_array_almost_equal(inv_std, expected_inv_std, decimal=4)
def _setup_model_loss(self, update_ops=None, num_classes=1): self.learning_rate_d = tf.placeholder( tf.float32, shape=[], name="learning_rate_placeholder_generator") self.learning_rate_g = tf.placeholder( tf.float32, shape=[], name="learning_rate_placeholder_discriminator") self.learning_rate_e = tf.placeholder( tf.float32, shape=[], name="learning_rate_placeholder_encoder") d_optimizer = self._optimizer(self.learning_rate_d, optname=self.cnf.get( 'optname', 'momentum'), **self.cnf.get('opt_kwargs', {'decay': 0.9})) g_optimizer = self._optimizer(self.learning_rate_g, optname=self.cnf.get( 'optname', 'momentum'), **self.cnf.get('opt_kwargs', {'decay': 0.9})) e_optimizer = self._optimizer(self.learning_rate_e, optname=self.cnf.get( 'optname', 'momentum'), **self.cnf.get('opt_kwargs', {'decay': 0.9})) # Get images split the batch across GPUs. assert self.cnf['batch_size_train'] % self.cnf.get( 'num_gpus', 1) == 0, ('Batch size must be divisible by number of GPUs') self.inputs = tf.placeholder(tf.float32, shape=( None, self.model.image_size[0], self.model.image_size[0], 3), name="input") global_step = tf.get_variable( 'global_step', [], initializer=tf.constant_initializer(0), trainable=False) capped_d_grads, capped_g_grads, capped_e_grads, self.tower_loss_d, self.tower_loss_g, self.tower_loss_e, self.tower_loss_d_real, self.tower_loss_d_fake, self.tower_loss_kl, self.tower_loss_like = self._process_tower_grads( d_optimizer, g_optimizer, e_optimizer, self.model, num_classes=num_classes, is_training=True, reuse=None) if self.gradient_multipliers is not None: with tf.name_scope('multiply_grads'): capped_d_grads = self._multiply_gradients( capped_d_grads, self.gradient_multipliers) apply_d_gradient_op = d_optimizer.apply_gradients( capped_d_grads, global_step=global_step) apply_g_gradient_op = g_optimizer.apply_gradients( capped_g_grads, global_step=global_step) apply_e_gradient_op = e_optimizer.apply_gradients( capped_e_grads, global_step=global_step) self.train_op_d = control_flow_ops.with_dependencies( [apply_d_gradient_op], self.tower_loss_d) self.train_op_g = control_flow_ops.with_dependencies( [apply_g_gradient_op], self.tower_loss_g) self.train_op_e = control_flow_ops.with_dependencies( [apply_e_gradient_op], self.tower_loss_e)
def testDelayedUpdateMovingVars(self): height, width = 3, 3 with self.test_session() as sess: image_shape = (10, height, width, 3) image_values = np.random.rand(*image_shape) expected_mean = np.mean(image_values, axis=(0, 1, 2)) expected_var = np.var(image_values, axis=(0, 1, 2)) images = tf.constant(image_values, shape=image_shape, dtype=tf.float32) output = tf.contrib.layers.batch_norm(images, decay=0.1) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # updates_ops are added to UPDATE_OPS collection. self.assertEquals(len(update_ops), 2) with tf.control_dependencies(update_ops): barrier = tf.no_op(name='barrier') output = control_flow_ops.with_dependencies([barrier], output) # Initialize all variables sess.run(tf.initialize_all_variables()) moving_mean = tf.contrib.framework.get_variables( 'BatchNorm/moving_mean')[0] moving_variance = tf.contrib.framework.get_variables( 'BatchNorm/moving_variance')[0] mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 3) self.assertAllClose(variance, [1] * 3) for _ in range(10): sess.run([output]) mean = moving_mean.eval() variance = moving_variance.eval() # After 10 updates with decay 0.1 moving_mean == expected_mean and # moving_variance == expected_var. self.assertAllClose(mean, expected_mean) self.assertAllClose(variance, expected_var)
def _get_train_ops(self, features, _): (_, _, losses, training_op) = gmm_ops.gmm( self._parse_tensor_or_dict(features), self._training_initial_clusters, self._num_clusters, self._random_seed, self._covariance_type, self._params) incr_step = tf.assign_add(tf.contrib.framework.get_global_step(), 1) loss = tf.reduce_sum(losses) training_op = with_dependencies([training_op, incr_step], loss) return training_op, loss
def get_sample_ndims(self, x, name="get_sample_ndims"): """Returns number of dimensions corresponding to iid draws ("sample"). Args: x: `Tensor`. name: `String`. The name to give this op. Returns: sample_ndims: `Tensor` (0D, `int32`). Raises: ValueError: if `sample_ndims` is calculated to be negative. """ with self._name_scope(name, values=[x]): ndims = self.get_ndims(x, name=name) if self._is_all_constant_helper(ndims, self.batch_ndims, self.event_ndims): ndims = tensor_util.constant_value(ndims) sample_ndims = (ndims - self._batch_ndims_static - self._event_ndims_static) if sample_ndims < 0: raise ValueError( "expected batch_ndims(%d) + event_ndims(%d) <= ndims(%d)" % (self._batch_ndims_static, self._event_ndims_static, ndims)) return ops.convert_to_tensor(sample_ndims, name="sample_ndims") else: with ops.name_scope(name="sample_ndims"): sample_ndims = ndims - self.batch_ndims - self.event_ndims if self.validate_args: sample_ndims = control_flow_ops.with_dependencies( [check_ops.assert_non_negative(sample_ndims)], sample_ndims) return sample_ndims
def _log_prob(self, x): x = control_flow_ops.with_dependencies([check_ops.assert_positive(x)] if self.validate_args else [], x) return (self.alpha * math_ops.log(self.beta) - math_ops.lgamma(self.alpha) - (self.alpha + 1.) * math_ops.log(x) - self.beta / x)
def _cdf(self, x): x = control_flow_ops.with_dependencies([check_ops.assert_positive(x)] if self.validate_args else [], x) # Note that igammac returns the upper regularized incomplete gamma # function Q(a, x), which is what we want for the CDF. return math_ops.igammac(self.alpha, self.beta / x)
def _variance(self): var = (math_ops.square(self.beta) / (math_ops.square(self.alpha - 1.) * (self.alpha - 2.))) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) return math_ops.select( self.alpha > 2., var, array_ops.fill(self.batch_shape(), nan, name="nan")) else: return control_flow_ops.with_dependencies([ check_ops.assert_less( constant_op.constant(2., dtype=self.dtype), self.alpha, message="variance not defined for components of alpha <= 2"), ], var)
def _check_integer(self, value): with ops.name_scope("check_integer", values=[value]): value = ops.convert_to_tensor(value, name="value") if not self.validate_args: return value dependencies = [distribution_util.assert_integer_form( value, message="value has non-integer components.")] return control_flow_ops.with_dependencies(dependencies, value)
def _check_counts(self, counts): """Check counts for proper shape, values, then return tensor version.""" counts = ops.convert_to_tensor(counts, name="counts_before_deps") if not self.validate_args: return counts return control_flow_ops.with_dependencies([ check_ops.assert_non_negative( counts, message="counts has negative components."), check_ops.assert_less_equal( counts, self._n, message="counts are not less than or equal to n."), distribution_util.assert_integer_form( counts, message="counts have non-integer components.")], counts)
def _assert_valid_counts(self, counts): """Check counts for proper shape, values, then return tensor version.""" counts = ops.convert_to_tensor(counts, name="counts") if not self.validate_args: return counts candidate_n = math_ops.reduce_sum(counts, reduction_indices=[-1]) return control_flow_ops.with_dependencies([ check_ops.assert_non_negative(counts), check_ops.assert_equal( self._n, candidate_n, message="counts do not sum to n"), distribution_util.assert_integer_form(counts)], counts)
def _assert_valid_alpha(self, alpha, validate_args): alpha = ops.convert_to_tensor(alpha, name="alpha") if not validate_args: return alpha return control_flow_ops.with_dependencies( [check_ops.assert_rank_at_least(alpha, 1), check_ops.assert_positive(alpha)], alpha)
